コード例 #1
0
  def _install_lzo_support_if_needed(self, params):
    hadoop_classpath_prefix = self._expand_hadoop_classpath_prefix(params.hadoop_classpath_prefix_template, params.config['configurations']['tez-site'])

    hadoop_lzo_dest_path = extract_path_component(hadoop_classpath_prefix, "hadoop-lzo-")
    if hadoop_lzo_dest_path:
      hadoop_lzo_file = os.path.split(hadoop_lzo_dest_path)[1]

      config = Script.get_config()
      file_url = urlparse.urljoin(config['hostLevelParams']['jdk_location'], hadoop_lzo_file)
      hadoop_lzo_dl_path = os.path.join(config["hostLevelParams"]["agentCacheDir"], hadoop_lzo_file)
      download_file(file_url, hadoop_lzo_dl_path)
      #This is for protection against configuration changes. It will infect every new destination with the lzo jar,
      # but since the classpath points to the jar directly we're getting away with it.
      if not os.path.exists(hadoop_lzo_dest_path):
        copy_file(hadoop_lzo_dl_path, hadoop_lzo_dest_path)
コード例 #2
0
See the License for the specific language governing permissions and
limitations under the License.

Ambari Agent

"""
import status_params

from resource_management.libraries.functions import format
from resource_management.libraries.functions.version import format_hdp_stack_version
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.script.script import Script

# server configurations
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()

stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)

stack_name = default("/hostLevelParams/stack_name", None)
current_version = default("/hostLevelParams/current_version", None)
component_directory = status_params.component_directory

# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
version = default("/commandParams/version", None)

# default parameters
zk_home = "/usr/hdp/2.3.2.0-2950/zookeeper"
zk_bin = "/usr/hdp/2.3.2.0-2950/zookeeper/bin"
コード例 #3
0
  def actionexecute(self, env):
    num_errors = 0

    # Parse parameters
    config = Script.get_config()

    repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
    repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
    template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu

    # Handle a SIGTERM and SIGINT gracefully
    signal.signal(signal.SIGTERM, self.abort_handler)
    signal.signal(signal.SIGINT, self.abort_handler)

    self.repository_version_id = None

    base_urls = []
    # Select dict that contains parameters
    try:
      if 'base_urls' in config['roleParams']:
        base_urls = json.loads(config['roleParams']['base_urls'])

      self.repository_version = config['roleParams']['repository_version']
      package_list = json.loads(config['roleParams']['package_list'])
      stack_id = config['roleParams']['stack_id']

      if 'repository_version_id' in config['roleParams']:
        self.repository_version_id = config['roleParams']['repository_version_id']
    except KeyError:
      pass

    # current stack information
    self.current_stack_version_formatted = None
    if 'stack_version' in config['hostLevelParams']:
      current_stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
      self.current_stack_version_formatted = format_stack_version(current_stack_version_unformatted)


    self.stack_name = Script.get_stack_name()
    if self.stack_name is None:
      raise Fail("Cannot determine the stack name")

    self.stack_root_folder = Script.get_stack_root()
    if self.stack_root_folder is None:
      raise Fail("Cannot determine the stack's root directory")

    if self.repository_version is None:
      raise Fail("Cannot determine the repository version to install")

    self.repository_version = self.repository_version.strip()

    # Install/update repositories
    self.current_repositories = []
    self.current_repo_files = set()

    # Enable base system repositories
    # We don't need that for RHEL family, because we leave all repos enabled
    # except disabled HDP* ones
    if OSCheck.is_suse_family():
      self.current_repositories.append('base')
    elif OSCheck.is_ubuntu_family():
      self.current_repo_files.add('base')

    Logger.info("Will install packages for repository version {0}".format(self.repository_version))

    if 0 == len(base_urls):
      Logger.warning("Repository list is empty. Ambari may not be managing the repositories for {0}.".format(self.repository_version))

    try:
      if 'repositoryFile' in config:
        create_repo_files(template, CommandRepository(config['repositoryFile']))
      else:
        append_to_file = False
        for url_info in base_urls:
          repo_name, repo_file = self.install_repository(url_info, append_to_file, template)
          self.current_repositories.append(repo_name)
          self.current_repo_files.add(repo_file)
          append_to_file = True

    except Exception, err:
      Logger.logger.exception("Cannot install repository files. Error: {0}".format(str(err)))
      num_errors += 1
コード例 #4
0
ファイル: params.py プロジェクト: gauravn7/ambari
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources

from resource_management.libraries.script.script import Script

# a map of the Ambari role to the component name
# for use with <stack-root>/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
    'SPARK_JOBHISTORYSERVER': 'spark-historyserver',
    'SPARK_CLIENT': 'spark-client',
    'SPARK_THRIFTSERVER': 'spark-thriftserver'
}

component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP,
                                                     "SPARK_CLIENT")

config = Script.get_config()
tmp_dir = Script.get_tmp_dir()

stack_name = default("/hostLevelParams/stack_name", None)
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
stack_version_formatted = format_stack_version(stack_version_unformatted)
host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)

# New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade
version = default("/commandParams/version", None)

# TODO! FIXME! Version check is not working as of today :
#   $ yum list installed | grep <stack-selector-tool>
#   <stack-selector-tool>.noarch                            2.2.1.0-2340.el6           @HDP-2.2
# And stack_version_formatted returned from hostLevelParams/stack_version is : 2.2.0.0
# Commenting out for time being
コード例 #5
0
from resource_management.libraries.script.script import Script

CONFIG = Script.get_config()
CONFIGS = CONFIG['configurations']
HOST_INFO = CONFIG['clusterHostInfo']
CASSANDRA_CONF = CONFIGS['cassandra-conf']

cassandra_conf_dir = '/etc/cassandra/conf'
cassandra_user = '******'

config_defaults = {
    'cross_node_timeout': False,
    'inter_dc_tcp_nodelay': False,
    'rpc_server_type':  'sync',
    'disk_failure_policy': 'stop',
    'authorizer': 'AllowAllAuthorizer',
    'tombstone_warn_threshold': 1000,
    'internode_compression': 'all',
    'truncate_request_timeout_in_ms': 60000,
    'cluster_name': 'Test Cluster',
    'read_request_timeout_in_ms': 5000,
    'ssl_storage_port': 7001,
    'listen_address': 'localhost',
    'request_scheduler': 'org.apache.cassandra.scheduler.NoScheduler',
    'range_request_timeout_in_ms': 10000,
    'hinted_handoff_enabled': True,
    'max_hint_window_in_ms': 10800000,
    'authenticator': 'AllowAllAuthenticator',
    'tombstone_failure_threshold': 100000,
    'commitlog_directory': '/var/lib/cassandra/commitlog',
    'column_index_size_in_kb': 64,
コード例 #6
0
    def actionexecute(self, env):
        num_errors = 0

        # Parse parameters
        config = Script.get_config()

        try:
            command_repository = CommandRepository(config['repositoryFile'])
        except KeyError:
            raise Fail(
                "The command repository indicated by 'repositoryFile' was not found"
            )

        repo_rhel_suse = config['configurations']['cluster-env'][
            'repo_suse_rhel_template']
        repo_ubuntu = config['configurations']['cluster-env'][
            'repo_ubuntu_template']
        template = repo_rhel_suse if OSCheck.is_redhat_family(
        ) or OSCheck.is_suse_family() else repo_ubuntu

        # Handle a SIGTERM and SIGINT gracefully
        signal.signal(signal.SIGTERM, self.abort_handler)
        signal.signal(signal.SIGINT, self.abort_handler)

        self.repository_version = command_repository.version_string

        # Select dict that contains parameters
        try:
            package_list = json.loads(config['roleParams']['package_list'])
            stack_id = config['roleParams']['stack_id']
        except KeyError:
            pass

        self.stack_name = Script.get_stack_name()
        if self.stack_name is None:
            raise Fail("Cannot determine the stack name")

        self.stack_root_folder = Script.get_stack_root()
        if self.stack_root_folder is None:
            raise Fail("Cannot determine the stack's root directory")

        if self.repository_version is None:
            raise Fail("Cannot determine the repository version to install")

        self.repository_version = self.repository_version.strip()

        try:
            if not command_repository.repositories:
                Logger.warning(
                    "Repository list is empty. Ambari may not be managing the repositories for {0}."
                    .format(self.repository_version))
            else:
                Logger.info(
                    "Will install packages for repository version {0}".format(
                        self.repository_version))
                new_repo_files = create_repo_files(template,
                                                   command_repository)
                self.repo_files.update(new_repo_files)
        except Exception, err:
            Logger.logger.exception(
                "Cannot install repository files. Error: {0}".format(str(err)))
            num_errors += 1
コード例 #7
0
 def install(self, env):
     self.install_packages(env)
     config = Script.get_config()
     self.configure(env)
コード例 #8
0
def get_packages(scope, service_name=None, component_name=None):
    """
  Gets the packages which should be used with the stack's stack-select tool for the
  specified service/component. Not all services/components are used with the stack-select tools,
  so those will return no packages.

  :param scope: the scope of the command
  :param service_name:  the service name, such as ZOOKEEPER
  :param component_name: the component name, such as ZOOKEEPER_SERVER
  :return:  the packages to use with stack-select or None
  """
    from resource_management.libraries.functions.default import default

    if scope not in _PACKAGE_SCOPES:
        raise Fail("The specified scope of {0} is not valid".format(scope))

    config = Script.get_config()

    if service_name is None or component_name is None:
        if 'role' not in config or 'serviceName' not in config:
            raise Fail(
                "Both the role and the service name must be included in the command in order to determine which packages to use with the stack-select tool"
            )

        service_name = config['serviceName']
        component_name = config['role']

    stack_name = default("/hostLevelParams/stack_name", None)
    if stack_name is None:
        raise Fail(
            "The stack name is not present in the command. Packages for stack-select tool cannot be loaded."
        )

    stack_packages_config = default(
        "/configurations/cluster-env/stack_packages", None)
    if stack_packages_config is None:
        raise Fail(
            "The stack packages are not defined on the command. Unable to load packages for the stack-select tool"
        )

    data = json.loads(stack_packages_config)

    if stack_name not in data:
        raise Fail(
            "Cannot find stack-select packages for the {0} stack".format(
                stack_name))

    stack_select_key = "stack-select"
    data = data[stack_name]
    if stack_select_key not in data:
        raise Fail(
            "There are no stack-select packages defined for this command for the {0} stack"
            .format(stack_name))

    # this should now be the dictionary of role name to package name
    data = data[stack_select_key]
    service_name = service_name.upper()
    component_name = component_name.upper()

    if service_name not in data:
        Logger.info(
            "Skipping stack-select on {0} because it does not exist in the stack-select package structure."
            .format(service_name))
        return None

    data = data[service_name]

    if component_name not in data:
        Logger.info(
            "Skipping stack-select on {0} because it does not exist in the stack-select package structure."
            .format(component_name))
        return None

    # this one scope is not an array, so transform it into one for now so we can
    # use the same code below
    packages = data[component_name][scope]
    if scope == PACKAGE_SCOPE_STACK_SELECT:
        packages = [packages]

    # grab the package name from the JSON and validate it against the packages
    # that the stack-select tool supports - if it doesn't support it, then try to find the legacy
    # package name if it exists
    supported_packages = get_supported_packages()
    for index, package in enumerate(packages):
        if not is_package_supported(package,
                                    supported_packages=supported_packages):
            if _PACKAGE_SCOPE_LEGACY in data[component_name]:
                legacy_package = data[component_name][_PACKAGE_SCOPE_LEGACY]
                Logger.info(
                    "The package {0} is not supported by this version of the stack-select tool, defaulting to the legacy package of {1}"
                    .format(package, legacy_package))

                # use the legacy package
                packages[index] = legacy_package
            else:
                raise Fail(
                    "The package {0} is not supported by this version of the stack-select tool."
                    .format(package))

    # transform the array bcak to a single element
    if scope == PACKAGE_SCOPE_STACK_SELECT:
        packages = packages[0]

    return packages
コード例 #9
0
def setup_spark(env, type, upgrade_type=None, action=None):
    import params
    config = Script.get_config()
    Directory([params.spark_pid_dir, params.spark_log_dir],
              owner=params.spark_user,
              group=params.user_group,
              mode=0775,
              create_parents=True)
    if type == 'server' and action == 'config':
        params.HdfsResource(params.spark_hdfs_user_dir,
                            type="directory",
                            action="create_on_execute",
                            owner=params.spark_user,
                            mode=0775)

        params.HdfsResource(None, action="execute")

        spark_history_dir = params.spark_history_dir
        hadoop_user = config['configurations']['cluster-env'][
            'user_group'] or 'hadoop'
        params.HdfsResource(spark_history_dir,
                            type="directory",
                            action="create_on_execute",
                            owner=params.spark_user,
                            group=hadoop_user,
                            mode=0775)

    PropertiesFile(
        format("{spark_conf}/spark-defaults.conf"),
        properties=params.config['configurations']['spark2-defaults'],
        key_value_delimiter=" ",
        owner=params.spark_user,
        group=params.spark_group,
        mode=0644)

    # create spark-env.sh in etc/conf dir
    File(
        os.path.join(params.spark_conf, 'spark-env.sh'),
        owner=params.spark_user,
        group=params.spark_group,
        content=InlineTemplate(params.spark_env_sh),
        mode=0644,
    )

    #create log4j.properties in etc/conf dir
    File(
        os.path.join(params.spark_conf, 'log4j.properties'),
        owner=params.spark_user,
        group=params.spark_group,
        content=params.spark_log4j_properties,
        mode=0644,
    )

    #create metrics.properties in etc/conf dir
    File(os.path.join(params.spark_conf, 'metrics.properties'),
         owner=params.spark_user,
         group=params.spark_group,
         content=InlineTemplate(params.spark_metrics_properties),
         mode=0644)

    if params.is_hive_installed:
        XmlConfig("hive-site.xml",
                  conf_dir=params.spark_conf,
                  configurations=params.spark_hive_properties,
                  owner=params.spark_user,
                  group=params.spark_group,
                  mode=0644)

    if params.has_spark_thriftserver:
        PropertiesFile(params.spark_thrift_server_conf_file,
                       properties=params.config['configurations']
                       ['spark2-thrift-sparkconf'],
                       owner=params.hive_user,
                       group=params.user_group,
                       key_value_delimiter=" ",
                       mode=0644)

    effective_version = params.version if upgrade_type is not None else params.stack_version_formatted
    if effective_version:
        effective_version = format_stack_version(effective_version)

    if params.spark_thrift_fairscheduler_content:
        # create spark-thrift-fairscheduler.xml
        File(os.path.join(params.spark_conf, "spark-thrift-fairscheduler.xml"),
             owner=params.spark_user,
             group=params.spark_group,
             mode=0755,
             content=InlineTemplate(params.spark_thrift_fairscheduler_content))
コード例 #10
0
    def actionexecute(self, env):
        num_errors = 0

        # Parse parameters
        config = Script.get_config()

        try:
            command_repository = CommandRepository(config['repositoryFile'])
        except KeyError:
            raise Fail(
                "The command repository indicated by 'repositoryFile' was not found"
            )

        # Handle a SIGTERM and SIGINT gracefully
        signal.signal(signal.SIGTERM, self.abort_handler)
        signal.signal(signal.SIGINT, self.abort_handler)

        self.repository_version = command_repository.version_string

        # Select dict that contains parameters
        try:
            package_list = json.loads(config['roleParams']['package_list'])
            stack_id = config['roleParams']['stack_id']
        except KeyError:
            pass

        self.stack_name = Script.get_stack_name()
        if self.stack_name is None:
            raise Fail("Cannot determine the stack name")

        self.stack_root_folder = Script.get_stack_root()
        if self.stack_root_folder is None:
            raise Fail("Cannot determine the stack's root directory")

        if self.repository_version is None:
            raise Fail("Cannot determine the repository version to install")

        self.repository_version = self.repository_version.strip()

        try:
            if not command_repository.items:
                Logger.warning(
                    "Repository list is empty. Ambari may not be managing the repositories for {0}."
                    .format(self.repository_version))
            else:
                Logger.info(
                    "Will install packages for repository version {0}".format(
                        self.repository_version))
                new_repo_files = Script.repository_util.create_repo_files()
                self.repo_files.update(new_repo_files)
        except Exception as err:
            Logger.logger.exception(
                "Cannot install repository files. Error: {0}".format(str(err)))
            num_errors += 1

        # Build structured output with initial values
        self.structured_output = {
            'package_installation_result': 'FAIL',
            'repository_version_id': command_repository.version_id
        }

        self.put_structured_out(self.structured_output)

        try:
            # check package manager non-completed transactions
            if self.repo_mgr.check_uncompleted_transactions():
                self.repo_mgr.print_uncompleted_transaction_hint()
                num_errors += 1
        except Exception as e:  # we need to ignore any exception
            Logger.warning(
                "Failed to check for uncompleted package manager transactions: "
                + str(e))

        if num_errors > 0:
            raise Fail("Failed to distribute repositories/install packages")

        # Initial list of versions, used to compute the new version installed
        self.old_versions = get_stack_versions(self.stack_root_folder)

        try:
            is_package_install_successful = False
            ret_code = self.install_packages(package_list)
            if ret_code == 0:
                self.structured_output[
                    'package_installation_result'] = 'SUCCESS'
                self.put_structured_out(self.structured_output)
                is_package_install_successful = True
            else:
                num_errors += 1
        except Exception as err:
            num_errors += 1
            Logger.logger.exception(
                "Could not install packages. Error: {0}".format(str(err)))

        # Provide correct exit code
        if num_errors > 0:
            raise Fail("Failed to distribute repositories/install packages")

        self._fix_default_links_for_current()
        # if installing a version of HDP that needs some symlink love, then create them
        if is_package_install_successful and 'actual_version' in self.structured_output:
            self._relink_configurations_with_conf_select(
                stack_id, self.structured_output['actual_version'])
コード例 #11
0
def get_packages(scope, service_name=None, component_name=None):
    """
  Gets the packages which should be used with the stack's stack-select tool for the
  specified service/component. Not all services/components are used with the stack-select tools,
  so those will return no packages.

  :param scope: the scope of the command
  :param service_name:  the service name, such as ZOOKEEPER
  :param component_name: the component name, such as ZOOKEEPER_SERVER
  :return:  the packages to use with stack-select or None
  """
    from resource_management.libraries.functions.default import default

    if scope not in _PACKAGE_SCOPES:
        raise Fail("The specified scope of {0} is not valid".format(scope))

    config = Script.get_config()

    if service_name is None or component_name is None:
        if 'role' not in config or 'serviceName' not in config:
            raise Fail(
                "Both the role and the service name must be included in the command in order to determine which packages to use with the stack-select tool"
            )

        service_name = config['serviceName']
        component_name = config['role']

    stack_name = default("/hostLevelParams/stack_name", None)
    if stack_name is None:
        raise Fail(
            "The stack name is not present in the command. Packages for stack-select tool cannot be loaded."
        )

    stack_packages_config = default(
        "/configurations/cluster-env/stack_packages", None)
    if stack_packages_config is None:
        raise Fail(
            "The stack packages are not defined on the command. Unable to load packages for the stack-select tool"
        )

    data = json.loads(stack_packages_config)

    if stack_name not in data:
        raise Fail(
            "Cannot find stack-select packages for the {0} stack".format(
                stack_name))

    stack_select_key = "stack-select"
    data = data[stack_name]
    if stack_select_key not in data:
        raise Fail(
            "There are no stack-select packages defined for this command for the {0} stack"
            .format(stack_name))

    # this should now be the dictionary of role name to package name
    data = data[stack_select_key]
    service_name = service_name.upper()
    component_name = component_name.upper()

    if service_name not in data:
        Logger.info(
            "Skipping stack-select on {0} because it does not exist in the stack-select package structure."
            .format(service_name))
        return None

    data = data[service_name]

    if component_name not in data:
        Logger.info(
            "Skipping stack-select on {0} because it does not exist in the stack-select package structure."
            .format(component_name))
        return None

    return data[component_name][scope]