def observer_namenode(action=None, format=False): if action == "configure": import params for fs_checkpoint_dir in params.fs_checkpoint_dirs: Directory(fs_checkpoint_dir, create_parents=True, cd_access="a", mode=0755, owner=params.hdfs_user, group=params.user_group) File(params.exclude_file_path, content=Template("exclude_hosts_list.j2"), owner=params.hdfs_user, group=params.user_group) if params.hdfs_include_file: File(params.include_file_path, content=Template("include_hosts_list.j2"), owner=params.hdfs_user, group=params.user_group) generate_logfeeder_input_config( 'hdfs', Template("input.config-hdfs.json.j2", extra_imports=[default])) elif action == "start" or action == "stop": import params service(action=action, name="observernamenode", user=params.hdfs_user, create_pid_dir=True, create_log_dir=True) elif action == "status": import status_params check_process_status(status_params.snamenode_pid_file)
def datanode(action=None): if action == "configure": import params Directory(params.dfs_domain_socket_dir, create_parents=True, mode=0751, owner=params.hdfs_user, group=params.user_group) # handle_mounted_dirs ensures that we don't create dfs data dirs which are temporary unavailable (unmounted), and intended to reside on a different mount. data_dir_to_mount_file_content = handle_mounted_dirs( create_dirs, params.dfs_data_dirs, params.data_dir_mount_file, params) # create a history file used by handle_mounted_dirs File(params.data_dir_mount_file, owner=params.hdfs_user, group=params.user_group, mode=0644, content=data_dir_to_mount_file_content) generate_logfeeder_input_config( 'hdfs', Template("input.config-hdfs.json.j2", extra_imports=[default])) elif action == "start" or action == "stop": import params service(action=action, name="datanode", user=params.hdfs_user, create_pid_dir=True, create_log_dir=True) elif action == "status": import params PidFiles(params.possible_datanode_pid_files).check_status()
def start(self, env, upgrade_type=None): import params env.set_params(params) self.configure(env) generate_logfeeder_input_config( 'ambari-infra', Template("input.config-ambari-infra.json.j2", extra_imports=[default])) setup_solr_znode_env() start_cmd = format('{solr_bindir}/solr start -cloud -noprompt -s {infra_solr_datadir} -Dsolr.kerberos.name.rules=\'{infra_solr_kerberos_name_rules}\' 2>&1') \ if params.security_enabled else format('{solr_bindir}/solr start -cloud -noprompt -s {infra_solr_datadir} 2>&1') check_process = format( "{sudo} test -f {infra_solr_pidfile} && {sudo} pgrep -F {infra_solr_pidfile}" ) piped_start_cmd = format('{start_cmd} | tee {infra_solr_log}' ) + '; (exit "${PIPESTATUS[0]}")' Execute(piped_start_cmd, environment={ 'SOLR_INCLUDE': format('{infra_solr_conf}/infra-solr-env.sh') }, user=params.infra_solr_user, not_if=check_process, logoutput=True)
def configure_static(env): import params env.set_params(params) generate_logfeeder_input_config('hdfs', Template("input.config-hdfs.json.j2", extra_imports=[default])) hdfs("zkfc_slave") utils.set_up_zkfc_security(params) pass
def configure(self, env): import params env.set_params(params) Directory( [params.log_dirs_list, '/etc/kafka', '/var/log/kafka', '/var/log/confluent/kafka'], owner='cp-kafka', group='confluent', mode=0775, create_parents=True) Directory([params.control_center_dir, params.kafka_log_dir], owner='cp-kafka', group='confluent', mode=0777, create_parents=True) File( '/etc/kafka/server.properties', owner='cp-kafka', group='confluent', mode=0644, content=InlineTemplate(params.kafka_content)) File( '/etc/kafka/log4j.properties', owner='cp-kafka', group='confluent', mode=0644, content=log4j_content) File( '/etc/sysconfig/kafka', owner='cp-kafka', group='confluent', mode=0644, content=InlineTemplate(params.kafka_env_content)) generate_logfeeder_input_config( 'kafka', Template("input.config-kafka.json.j2", extra_imports=[default]))
def configure(self, env): import params import status_params env.set_params(params) env.set_params(status_params) # create the log, pid, conf dirs if not already present Directory([ status_params.nifi_pid_dir, params.nifi_node_log_dir, params.nifi_config_dir ], owner=params.nifi_user, group=params.nifi_group, create_parents=True) ca_json = os.path.join(params.nifi_config_dir, 'nifi-certificate-authority.json') ca_dict = nifi_toolkit_util_common.load(ca_json) nifi_toolkit_util_common.overlay(ca_dict, params.nifi_ca_config) nifi_toolkit_util_common.dump(ca_json, ca_dict, params.nifi_user, params.nifi_group) generate_logfeeder_input_config( 'nifi', Template("input.config-nifi.json.j2", extra_imports=[default])) Directory([params.nifi_config_dir], owner=params.nifi_user, group=params.nifi_group, create_parents=True, recursive_ownership=True)
def flink(name=None): import params import os Directory(params.log_dir, owner=params.flink_user, group=params.user_group, mode=0777, create_parents = True, cd_access="a", ) Directory([params.pid_dir], owner=params.flink_user, group=params.user_group, create_parents = True, cd_access="a", mode=0755, ) configurations = params.config['configurations']['flink-site'] File(format("{conf_dir}/flink-conf.yaml"), content=yaml_config_template(configurations), owner=params.flink_user, group=params.user_group ) generate_logfeeder_input_config('flink', Template("input.config-flink.json.j2", extra_imports=[default])) if params.has_metric_collector: File(format("{conf_dir}/flink-metrics2.properties"), owner=params.flink_user, group=params.user_group, content=Template("flink-metrics2.properties.j2") ) if params.security_enabled: TemplateConfig(format("{conf_dir}/flink_jaas.conf"), owner=params.flink_user, mode=0644 ) if params.stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted): TemplateConfig(format("{conf_dir}/client_jaas.conf"), owner=params.flink_user, mode=0644 ) else: File( format("{conf_dir}/flink_jaas.conf"), action="delete" ) File( format("{conf_dir}/client_jaas.conf"), action="delete" )
def setup_historyserver(): import params if params.yarn_log_aggregation_enabled: params.HdfsResource(params.yarn_nm_app_log_dir, action="create_on_execute", type="directory", owner=params.yarn_user, group=params.user_group, mode=01777, recursive_chmod=True) # create the /tmp folder with proper permissions if it doesn't exist yet if params.entity_file_history_directory.startswith('/tmp'): params.HdfsResource( params.hdfs_tmp_dir, action="create_on_execute", type="directory", owner=params.hdfs_user, mode=0777, ) params.HdfsResource(params.entity_file_history_directory, action="create_on_execute", type="directory", owner=params.yarn_user, group=params.user_group) params.HdfsResource("/mapred", type="directory", action="create_on_execute", owner=params.mapred_user) params.HdfsResource("/mapred/system", type="directory", action="create_on_execute", owner=params.hdfs_user) params.HdfsResource(params.mapreduce_jobhistory_done_dir, type="directory", action="create_on_execute", owner=params.mapred_user, group=params.user_group, change_permissions_for_parents=True, mode=0777) params.HdfsResource(None, action="execute") Directory( params.jhs_leveldb_state_store_dir, owner=params.mapred_user, group=params.user_group, create_parents=True, cd_access="a", recursive_ownership=True, ) generate_logfeeder_input_config( 'mapreduce2', Template("input.config-mapreduce2.json.j2", extra_imports=[default]))
def configure(self, env): import params Directory(params.jn_edits_dirs, create_parents = True, cd_access="a", owner=params.hdfs_user, group=params.user_group ) env.set_params(params) generate_logfeeder_input_config('hdfs', Template("input.config-hdfs.json.j2", extra_imports=[default])) hdfs() pass
def setup_metastore(): import params if params.hive_metastore_site_supported: hivemetastore_site_config = get_config("hivemetastore-site") if hivemetastore_site_config: XmlConfig("hivemetastore-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations'] ['hivemetastore-site'], configuration_attributes=params. config['configurationAttributes']['hivemetastore-site'], owner=params.hive_user, group=params.user_group, mode=0600) File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hivemetastore.properties.j2"), mode=0600) File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh')) if params.hive_repl_cmrootdir is not None and params.hive_repl_cmrootdir.strip( ) != "": params.HdfsResource(params.hive_repl_cmrootdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=01777) if params.hive_repl_rootdir is not None and params.hive_repl_rootdir.strip( ) != "": params.HdfsResource(params.hive_repl_rootdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=0700) params.HdfsResource(None, action="execute") generate_logfeeder_input_config( 'hive', Template("input.config-hive.json.j2", extra_imports=[default]))
def registry(env, upgrade_type=None): import params ensure_base_directories() #Logger.info(format("Effective stack version: {effective_version}")) File(format("{conf_dir}/registry-env.sh"), owner=params.registry_user, content=InlineTemplate(params.registry_env_sh_template)) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') Directory( [params.jar_storage], owner=params.registry_user, group=params.user_group, create_parents=True, cd_access="a", mode=0755, ) File(os.path.join(params.limits_conf_dir, 'registry.conf'), owner='root', group='root', mode=0644, content=Template("registry.conf.j2")) File(format("{conf_dir}/registry.yaml"), content=Template("registry.yaml.j2"), owner=params.registry_user, group=params.user_group, mode=0600) generate_logfeeder_input_config( 'registry', Template("input.config-registry.json.j2", extra_imports=[default])) if not os.path.islink(params.registry_managed_log_dir): Link(params.registry_managed_log_dir, to=params.registry_log_dir) download_database_connector_if_needed()
def configure(self, env, isInstall=False, is_starting = False): import params import status_params env.set_params(params) env.set_params(status_params) #create the log, pid, conf dirs if not already present nifi_dirs = [status_params.nifi_pid_dir, params.nifi_internal_dir, params.nifi_database_dir, params.nifi_flowfile_repo_dir, params.nifi_provenance_repo_dir_default, params.nifi_config_dir, params.nifi_flow_config_dir, params.nifi_state_dir, params.lib_dir] nifi_dirs.extend(params.nifi_content_repo_dirs) Directory(nifi_dirs, owner=params.nifi_user, group=params.nifi_group, create_parents=True, recursive_ownership=True, cd_access='a') # On some OS this folder may not exist, so we will create it before pushing files there Directory(params.limits_conf_dir, create_parents = True, owner='root', group='root' ) File(os.path.join(params.limits_conf_dir, 'nifi.conf'), owner='root', group='root', mode=0644, content=Template("nifi.conf.j2") ) #determine whether or not a cluster already exists based on zookeeper entries and determine if this is the first start of this node #if so authorizations and flow file will not be writen if not sudo.path_isfile(params.nifi_flow_config_dir+'/flow.xml.gz') and nifi_toolkit_util_common.existing_cluster(params): params.is_additional_node = True #write configurations self.write_configurations(params, is_starting) generate_logfeeder_input_config('nifi', Template("input.config-nifi.json.j2", extra_imports=[default])) # if this is not an additional node being added to an existing cluster write out flow.xml.gz to internal dir only if AMS installed (must be writable by Nifi) # and only during first install. It is used to automate setup of Ambari metrics reporting task in Nifi if not params.is_additional_node: if params.metrics_collector_host and params.nifi_ambari_reporting_enabled and not sudo.path_isfile(params.nifi_flow_config_dir+'/flow.xml.gz'): Execute('echo "First time setup so generating flow.xml.gz" >> ' + params.nifi_node_log_file, user=params.nifi_user) flow_content=InlineTemplate(params.nifi_flow_content) File(format("{params.nifi_flow_config_dir}/flow.xml"), content=flow_content, owner=params.nifi_user, group=params.nifi_group, mode=0600) Execute(format("cd {params.nifi_flow_config_dir}; gzip flow.xml;"), user=params.nifi_user)
def nfsgateway(action=None, format=False): import params if action== "start": prepare_rpcbind() if action == "configure": Directory(params.nfs_file_dump_dir, owner = params.hdfs_user, group = params.user_group, ) generate_logfeeder_input_config('hdfs', Template("input.config-hdfs.json.j2", extra_imports=[default])) elif action == "start" or action == "stop": service( action=action, name="nfs3", user=params.root_user, create_pid_dir=True, create_log_dir=True )
def registry(env, upgrade_type=None): import params ensure_base_directories() File(format("{conf_dir}/registry-env.sh"), owner=params.registry_user, content=InlineTemplate(params.registry_env_sh_template)) Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') Directory( [params.jar_storage], owner=params.registry_user, group=params.user_group, create_parents=True, cd_access="a", mode=0755, ) File(os.path.join(params.limits_conf_dir, 'registry.conf'), owner='root', group='root', mode=0644, content=Template("registry.conf.j2")) File(format("{conf_dir}/registry.yaml"), content=Template("registry.yaml.j2"), owner=params.registry_user, group=params.user_group, mode=0600) generate_logfeeder_input_config( 'registry', Template("input.config-registry.json.j2", extra_imports=[default])) if not os.path.islink(params.registry_managed_log_dir): Link(params.registry_managed_log_dir, to=params.registry_log_dir)
def setup_livy(env, type, upgrade_type=None, action=None): import params Directory([params.livy2_pid_dir, params.livy2_log_dir], owner=params.livy2_user, group=params.user_group, mode=0775, cd_access='a', create_parents=True) if type == 'server' and action == 'config': params.HdfsResource(params.livy2_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.livy2_user, mode=0775) params.HdfsResource(None, action="execute") params.HdfsResource(params.livy2_recovery_dir, type="directory", action="create_on_execute", owner=params.livy2_user, mode=0700) params.HdfsResource(None, action="execute") generate_logfeeder_input_config( 'spark2', Template("input.config-spark2.json.j2", extra_imports=[default])) # create livy-env.sh in etc/conf dir File( os.path.join(params.livy2_conf, 'livy-env.sh'), owner=params.livy2_user, group=params.livy2_group, content=InlineTemplate(params.livy2_env_sh), mode=0644, ) # create livy.conf in etc/conf dir PropertiesFile( format("{livy2_conf}/livy.conf"), properties=params.config['configurations']['livy2-conf'], key_value_delimiter=" ", owner=params.livy2_user, group=params.livy2_group, ) # create log4j.properties in etc/conf dir File( os.path.join(params.livy2_conf, 'log4j.properties'), owner=params.livy2_user, group=params.livy2_group, content=params.livy2_log4j_properties, mode=0644, ) # create spark-blacklist.properties in etc/conf dir File( os.path.join(params.livy2_conf, 'spark-blacklist.conf'), owner=params.livy2_user, group=params.livy2_group, content=params.livy2_spark_blacklist_properties, mode=0644, ) Directory( params.livy2_logs_dir, owner=params.livy2_user, group=params.livy2_group, mode=0755, )
def kms(upgrade_type=None): import params if params.has_ranger_admin: Directory(params.kms_conf_dir, owner=params.kms_user, group=params.kms_group, create_parents=True) Directory("/etc/security/serverKeys", create_parents=True, cd_access="a") Directory("/etc/ranger/kms", create_parents=True, cd_access="a") copy_jdbc_connector(params.kms_home) File( format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content=DownloadSource( format("{jdk_location}/{check_db_connection_jar_name}")), mode=0644, ) cp = format("{check_db_connection_jar}") if params.db_flavor.lower() == 'sqla': cp = cp + os.pathsep + format( "{kms_home}/ews/webapp/lib/sajdbc4.jar") else: path_to_jdbc = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}") if not os.path.isfile(path_to_jdbc): path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + \ params.default_connectors_map[ params.db_flavor.lower()] if params.db_flavor.lower() in params.default_connectors_map else None if not os.path.isfile(path_to_jdbc): path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + "*" error_message = "Error! Sorry, but we can't find jdbc driver with default name " + \ params.default_connectors_map[params.db_flavor] + \ " in ranger kms lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'" Logger.error(error_message) cp = cp + os.pathsep + path_to_jdbc db_connection_check_command = format( "{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_kms_jdbc_connection_url}' {db_user} {db_password!p} {ranger_kms_jdbc_driver}" ) env_dict = {} if params.db_flavor.lower() == 'sqla': env_dict = {'LD_LIBRARY_PATH': params.ld_library_path} Execute(db_connection_check_command, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10, environment=env_dict) if params.xa_audit_db_is_enabled and params.driver_source is not None and not params.driver_source.endswith( "/None"): if params.xa_previous_jdbc_jar and os.path.isfile( params.xa_previous_jdbc_jar): File(params.xa_previous_jdbc_jar, action='delete') File(params.downloaded_connector_path, content=DownloadSource(params.driver_source), mode=0644) Execute(('cp', '--remove-destination', params.downloaded_connector_path, params.driver_target), path=["/bin", "/usr/bin/"], sudo=True) File(params.driver_target, mode=0644) Directory(os.path.join(params.kms_home, 'ews', 'webapp', 'WEB-INF', 'classes', 'lib'), mode=0755, owner=params.kms_user, group=params.kms_group) Execute(('cp', format('{kms_home}/ranger-kms-initd'), '/etc/init.d/ranger-kms'), not_if=format('ls /etc/init.d/ranger-kms'), only_if=format('ls {kms_home}/ranger-kms-initd'), sudo=True) File('/etc/init.d/ranger-kms', mode=0755) Directory( format('{kms_home}/'), owner=params.kms_user, group=params.kms_group, recursive_ownership=True, ) Directory(params.ranger_kms_pid_dir, mode=0755, owner=params.kms_user, group=params.user_group, cd_access="a", create_parents=True) Directory(params.kms_log_dir, owner=params.kms_user, group=params.kms_group, cd_access='a', create_parents=True, mode=0755) generate_logfeeder_input_config( 'ranger-kms', Template("input.config-ranger-kms.json.j2", extra_imports=[default])) File(format('{kms_conf_dir}/ranger-kms-env.sh'), content=InlineTemplate(params.kms_env_content), owner=params.kms_user, group=params.kms_group, mode=0755) Execute(('ln', '-sf', format('{kms_home}/ranger-kms'), '/usr/bin/ranger-kms'), not_if=format('ls /usr/bin/ranger-kms'), only_if=format('ls {kms_home}/ranger-kms'), sudo=True) File('/usr/bin/ranger-kms', mode=0755) Execute(('ln', '-sf', format('{kms_home}/ranger-kms'), '/usr/bin/ranger-kms-services.sh'), not_if=format('ls /usr/bin/ranger-kms-services.sh'), only_if=format('ls {kms_home}/ranger-kms'), sudo=True) File('/usr/bin/ranger-kms-services.sh', mode=0755) Execute(('ln', '-sf', format('{kms_home}/ranger-kms-initd'), format('{kms_home}/ranger-kms-services.sh')), not_if=format('ls {kms_home}/ranger-kms-services.sh'), only_if=format('ls {kms_home}/ranger-kms-initd'), sudo=True) File(format('{kms_home}/ranger-kms-services.sh'), mode=0755) do_keystore_setup(params.credential_provider_path, params.jdbc_alias, params.db_password) do_keystore_setup(params.credential_provider_path, params.masterkey_alias, params.kms_master_key_password) if params.stack_support_kms_hsm and params.enable_kms_hsm: do_keystore_setup(params.credential_provider_path, params.hms_partition_alias, unicode(params.hms_partition_passwd)) if params.stack_supports_ranger_kms_ssl and params.ranger_kms_ssl_enabled: do_keystore_setup(params.ranger_kms_cred_ssl_path, params.ranger_kms_ssl_keystore_alias, params.ranger_kms_ssl_passwd) # remove plain-text password from xml configs dbks_site_copy = {} dbks_site_copy.update(params.config['configurations']['dbks-site']) for prop in params.dbks_site_password_properties: if prop in dbks_site_copy: dbks_site_copy[prop] = "_" XmlConfig( "dbks-site.xml", conf_dir=params.kms_conf_dir, configurations=dbks_site_copy, configuration_attributes=params.config['configurationAttributes'] ['dbks-site'], owner=params.kms_user, group=params.kms_group, mode=0644) ranger_kms_site_copy = {} ranger_kms_site_copy.update( params.config['configurations']['ranger-kms-site']) if params.stack_supports_ranger_kms_ssl: # remove plain-text password from xml configs for prop in params.ranger_kms_site_password_properties: if prop in ranger_kms_site_copy: ranger_kms_site_copy[prop] = "_" XmlConfig( "ranger-kms-site.xml", conf_dir=params.kms_conf_dir, configurations=ranger_kms_site_copy, configuration_attributes=params.config['configurationAttributes'] ['ranger-kms-site'], owner=params.kms_user, group=params.kms_group, mode=0644) XmlConfig( "kms-site.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations']['kms-site'], configuration_attributes=params.config['configurationAttributes'] ['kms-site'], owner=params.kms_user, group=params.kms_group, mode=0644) File(os.path.join(params.kms_conf_dir, "kms-log4j.properties"), owner=params.kms_user, group=params.kms_group, content=InlineTemplate(params.kms_log4j), mode=0644) # core-site.xml linking required by setup for HDFS encryption XmlConfig( "core-site.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations']['core-site'], configuration_attributes=params.config['configurationAttributes'] ['core-site'], owner=params.kms_user, group=params.kms_group, mode=0644, xml_include_file=params.mount_table_xml_inclusion_file_full_path) if params.mount_table_content: File(params.mount_table_xml_inclusion_file_full_path, owner=params.kms_user, group=params.kms_group, content=params.mount_table_content, mode=0644)
def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, upgrade_suspended=False, env=None): if action is None: raise Fail('"action" parameter is required for function namenode().') if action in ["start", "stop"] and hdfs_binary is None: raise Fail( '"hdfs_binary" parameter is required for function namenode().') if action == "configure": import params #we need this directory to be present before any action(HA manual steps for #additional namenode) create_name_dirs(params.dfs_name_dir) generate_logfeeder_input_config( 'hdfs', Template("input.config-hdfs.json.j2", extra_imports=[default])) # set up failover / secure zookeper ACLs, this feature is supported from HDP 2.6 ownwards set_up_zkfc_security(params) elif action == "start": Logger.info("Called service {0} with upgrade_type: {1}".format( action, str(upgrade_type))) setup_ranger_hdfs(upgrade_type=upgrade_type) import params File(params.exclude_file_path, content=Template("exclude_hosts_list.j2"), owner=params.hdfs_user, group=params.user_group) if do_format and not params.hdfs_namenode_format_disabled: format_namenode() pass if params.dfs_ha_enabled and \ len(params.dfs_ha_namenode_active) > 0 and \ params.hostname not in params.dfs_ha_namenode_active: # if the current host is the standby NameNode in an HA deployment # run the bootstrap command, to start the NameNode in standby mode # this requires that the active NameNode is already up and running, # so this execute should be re-tried upon failure, up to a timeout success = bootstrap_standby_namenode(params) if not success: raise Fail("Could not bootstrap standby namenode") if upgrade_type == constants.UPGRADE_TYPE_ROLLING and params.dfs_ha_enabled: # Most likely, ZKFC is up since RU will initiate the failover command. However, if that failed, it would have tried # to kill ZKFC manually, so we need to start it if not already running. safe_zkfc_op(action, env) summary = upgrade_summary.get_upgrade_summary() is_downgrade_allowed = summary is not None and summary.is_downgrade_allowed is_switch_bits = summary is not None and summary.is_switch_bits options = "" if is_switch_bits: Logger.info( "The {0} switches the binaries only. No options are used to restart NameNode." .format(summary.direction)) else: if upgrade_type == constants.UPGRADE_TYPE_ROLLING: if params.upgrade_direction == Direction.UPGRADE: options = "-rollingUpgrade started" elif params.upgrade_direction == Direction.DOWNGRADE: options = "" elif upgrade_type == constants.UPGRADE_TYPE_NON_ROLLING: is_previous_image_dir = is_previous_fs_image() Logger.info( "Previous file system image dir present is {0}".format( str(is_previous_image_dir))) if params.upgrade_direction == Direction.UPGRADE: if is_downgrade_allowed: options = "-rollingUpgrade started" else: # if we are HA, then -upgrade needs to be called for the active NN, # then -bootstrapStandby on the other, followed by normal daemon # if we are NOT HA, then -upgrade needs to be called on the lone NN if params.dfs_ha_enabled: name_service = get_name_service_by_hostname( params.hdfs_site, params.hostname) any_active = is_there_any_active_nn(name_service) if any_active: if not bootstrap_standby_namenode( params, use_path=True, run_if_present=True): raise Fail( "Could not bootstrap this namenode of an Express upgrade" ) options = "" # we're bootstrapped, no other work needs to happen for the daemon else: options = "-upgrade" # no other are active, so this host's NN is the first else: options = "-upgrade" # non-HA marker = os.path.exists( namenode_upgrade.get_upgrade_in_progress_marker()) if options == "-upgrade" and upgrade_suspended is True and marker is True: Logger.info( "The NameNode is currently upgrading. No options will be passed to startup" ) options = "" elif params.upgrade_direction == Direction.DOWNGRADE: options = "" elif upgrade_type == constants.UPGRADE_TYPE_HOST_ORDERED: # nothing special to do for HOU - should be very close to a normal restart pass elif upgrade_type is None and upgrade_suspended is True: # the rollingUpgrade flag must be passed in during a suspended upgrade when starting NN if os.path.exists( namenode_upgrade.get_upgrade_in_progress_marker()): if is_downgrade_allowed: options = "-rollingUpgrade started" else: options = "-upgrade" else: Logger.info( "The NameNode upgrade marker file {0} does not exist, yet an upgrade is currently suspended. " "Assuming that the upgrade of NameNode has not occurred yet." .format( namenode_upgrade.get_upgrade_in_progress_marker())) Logger.info("Options for start command are: {0}".format(options)) service(action="start", name="namenode", user=params.hdfs_user, options=options, create_pid_dir=True, create_log_dir=True) if params.security_enabled: Execute(format( "{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}" ), user=params.hdfs_user) name_service = get_name_service_by_hostname(params.hdfs_site, params.hostname) # ___Scenario___________|_Expected safemode state__|_Wait for safemode OFF____| # no-HA | ON -> OFF | Yes | # HA and active | ON -> OFF | Yes | # HA and standby | no change | No | # RU with HA on active | ON -> OFF | Yes | # RU with HA on standby | ON -> OFF | Yes | # EU with HA on active | ON -> OFF | No | # EU with HA on standby | ON -> OFF | No | # EU non-HA | ON -> OFF | No | # because we do things like create directories after starting NN, # the vast majority of the time this should be True - it should only # be False if this is HA and we are the Standby NN ensure_safemode_off = True # True if this is the only NameNode (non-HA) or if its the Active one in HA is_active_namenode = True if params.dfs_ha_enabled: Logger.info( "Waiting for the NameNode to broadcast whether it is Active or Standby..." ) if is_this_namenode_active(name_service) is False: # we are the STANDBY NN is_active_namenode = False # we are the STANDBY NN and this restart is not part of an upgrade if upgrade_type is None: ensure_safemode_off = False # During an Express Upgrade, NameNode will not leave SafeMode until the DataNodes are started, # so always disable the Safemode check if upgrade_type == constants.UPGRADE_TYPE_NON_ROLLING: ensure_safemode_off = False # some informative logging separate from the above logic to keep things a little cleaner if ensure_safemode_off: Logger.info( "Waiting for this NameNode to leave Safemode due to the following conditions: HA: {0}, isActive: {1}, upgradeType: {2}" .format(params.dfs_ha_enabled, is_active_namenode, upgrade_type)) else: Logger.info( "Skipping Safemode check due to the following conditions: HA: {0}, isActive: {1}, upgradeType: {2}" .format(params.dfs_ha_enabled, is_active_namenode, upgrade_type)) # wait for Safemode to end if ensure_safemode_off: if params.rolling_restart and params.rolling_restart_safemode_exit_timeout: calculated_retries = int( params.rolling_restart_safemode_exit_timeout) / 30 wait_for_safemode_off(hdfs_binary, afterwait_sleep=30, retries=calculated_retries, sleep_seconds=30) else: wait_for_safemode_off(hdfs_binary) # Always run this on the "Active" NN unless Safemode has been ignored # in the case where safemode was ignored (like during an express upgrade), then # NN will be in SafeMode and cannot have directories created if is_active_namenode and ensure_safemode_off: create_hdfs_directories(name_service) create_ranger_audit_hdfs_directories() else: Logger.info( "Skipping creation of HDFS directories since this is either not the Active NameNode or we did not wait for Safemode to finish." ) elif action == "stop": import params service(action="stop", name="namenode", user=params.hdfs_user) elif action == "status": import status_params check_process_status(status_params.namenode_pid_file) elif action == "decommission": decommission() elif action == "refresh_nodes": refresh_nodes()
def hive_interactive(name=None): import params MB_TO_BYTES = 1048576 # if warehouse directory is in DFS if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse( params.default_fs).scheme: # Create Hive Metastore Warehouse Dir params.HdfsResource(params.hive_metastore_warehouse_external_dir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=01777) params.HdfsResource(params.hive_metastore_warehouse_dir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=0700) # create directories for llap package pkg_dir = '/user/' + params.hive_user + '/.yarn' for dir in [pkg_dir, pkg_dir + '/package', pkg_dir + '/package/LLAP']: # hdfsresouces handles parent creation badly params.HdfsResource(dir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=01755) if not is_empty(params.tez_hook_proto_base_directory): params.HdfsResource(params.tez_hook_proto_base_directory, type="directory", action="create_on_execute", owner=params.hive_user, mode=01755) if not is_empty(params.hive_hook_proto_base_directory): params.HdfsResource(params.hive_hook_proto_base_directory, type="directory", action="create_on_execute", owner=params.hive_user, mode=01777) dag_meta = params.tez_hook_proto_base_directory + "dag_meta" params.HdfsResource(dag_meta, type="directory", action="create_on_execute", owner=params.hive_user, mode=01777) dag_data = params.tez_hook_proto_base_directory + "dag_data" params.HdfsResource(dag_data, type="directory", action="create_on_execute", owner=params.hive_user, mode=01777) app_data = params.tez_hook_proto_base_directory + "app_data" params.HdfsResource(app_data, type="directory", action="create_on_execute", owner=params.hive_user, mode=01777) else: Logger.info( format( "Not creating warehouse directory '{hive_metastore_warehouse_dir}', as the location is not in DFS." )) # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=params.hive_hdfs_user_mode) params.HdfsResource(None, action="execute") # list of properties that should be excluded from the config # this approach is a compromise against adding a dedicated config # type for hive_server_interactive or needed config groups on a # per component basis exclude_list = ['hive.enforce.bucketing', 'hive.enforce.sorting'] # List of configs to be excluded from hive2 client, but present in Hive2 server. exclude_list_for_hive2_client = [ 'javax.jdo.option.ConnectionPassword', 'hadoop.security.credential.provider.path' ] Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list)) for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) ''' As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml, we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml. ''' merged_hive_interactive_site = {} merged_hive_interactive_site.update(params.hive_site_config) merged_hive_interactive_site.update( params.config['configurations']['hive-interactive-site']) for item in exclude_list: if item in merged_hive_interactive_site.keys(): del merged_hive_interactive_site[item] merged_hive_interactive_site[ 'hive.llap.daemon.vcpus.per.instance'] = format( merged_hive_interactive_site['hive.llap.daemon.vcpus.per.instance'] ) merged_hive_interactive_site[ 'hive.server2.active.passive.ha.enable'] = str( params.hive_server_interactive_ha).lower() ''' Config 'hive.llap.io.memory.size' calculated value in stack_advisor is in MB as of now. We need to convert it to bytes before we write it down to config file. ''' if 'hive.llap.io.memory.size' in merged_hive_interactive_site.keys(): hive_llap_io_mem_size_in_mb = merged_hive_interactive_site.get( "hive.llap.io.memory.size") hive_llap_io_mem_size_in_bytes = long( hive_llap_io_mem_size_in_mb) * MB_TO_BYTES merged_hive_interactive_site[ 'hive.llap.io.memory.size'] = hive_llap_io_mem_size_in_bytes Logger.info( "Converted 'hive.llap.io.memory.size' value from '{0} MB' to '{1} Bytes' before writing " "it to config file.".format(hive_llap_io_mem_size_in_mb, hive_llap_io_mem_size_in_bytes)) ''' Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook', which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed. ''' # Generate atlas-application.properties.xml file if params.enable_atlas_hook and params.stack_supports_atlas_hook_for_hive_interactive: Logger.info("Setup for Atlas Hive2 Hook started.") atlas_hook_filepath = os.path.join( params.hive_server_interactive_conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) Logger.info("Setup for Atlas Hive2 Hook done.") else: # Required for HDP 2.5 stacks Logger.info( "Skipping setup for Atlas Hook, as it is disabled/ not supported.") remove_atlas_hook_if_exists(merged_hive_interactive_site) ''' As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml, we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml. ''' merged_tez_interactive_site = {} if 'tez-site' in params.config['configurations']: merged_tez_interactive_site.update( params.config['configurations']['tez-site']) Logger.info( "Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'." ) else: Logger.error( "Tez's 'tez-site' couldn't be retrieved from passed-in configurations." ) merged_tez_interactive_site.update( params.config['configurations']['tez-interactive-site']) XmlConfig("tez-site.xml", conf_dir=params.tez_interactive_conf_dir, configurations=merged_tez_interactive_site, configuration_attributes=params.config['configurationAttributes'] ['tez-interactive-site'], owner=params.tez_interactive_user, group=params.user_group, mode=0664) ''' Merge properties from hiveserver2-interactive-site into hiveserver2-site ''' merged_hiveserver2_interactive_site = {} if 'hiveserver2-site' in params.config['configurations']: merged_hiveserver2_interactive_site.update( params.config['configurations']['hiveserver2-site']) Logger.info( "Retrieved 'hiveserver2-site' for merging with 'hiveserver2-interactive-site'." ) else: Logger.error( "'hiveserver2-site' couldn't be retrieved from passed-in configurations." ) merged_hiveserver2_interactive_site.update( params.config['configurations']['hiveserver2-interactive-site']) # Create config files under hive_server_interactive_conf_dir: # hive-site.xml # hive-env.sh # llap-daemon-log4j2.properties # llap-cli-log4j2.properties # hive-log4j2.properties # hive-exec-log4j2.properties # beeline-log4j2.properties hive_server_interactive_conf_dir = params.hive_server_interactive_conf_dir mode_identified = 0600 merged_hive_interactive_site = update_credential_provider_path( merged_hive_interactive_site, 'hive-site', os.path.join(conf_dir, 'hive-site.jceks'), params.hive_user, params.user_group) XmlConfig("hive-site.xml", conf_dir=hive_server_interactive_conf_dir, configurations=merged_hive_interactive_site, configuration_attributes=params.config['configurationAttributes'] ['hive-interactive-site'], owner=params.hive_user, group=params.user_group, mode=0644) XmlConfig("hiveserver2-site.xml", conf_dir=hive_server_interactive_conf_dir, configurations=merged_hiveserver2_interactive_site, configuration_attributes=params.config['configurationAttributes'] ['hiveserver2-interactive-site'], owner=params.hive_user, group=params.user_group, mode=mode_identified) File(format("{hive_server_interactive_conf_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, mode=0755, content=InlineTemplate(params.hive_interactive_env_sh_template)) llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties' File(format( "{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.llap_daemon_log4j)) llap_cli_log4j2_filename = 'llap-cli-log4j2.properties' File(format( "{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.llap_cli_log4j2)) hive_log4j2_filename = 'hive-log4j2.properties' File(format("{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.hive_log4j2)) hive_exec_log4j2_filename = 'hive-exec-log4j2.properties' File(format( "{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.hive_exec_log4j2)) beeline_log4j2_filename = 'beeline-log4j2.properties' File( format("{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.beeline_log4j2)) XmlConfig("beeline-site.xml", conf_dir=conf_dir, configurations=params.beeline_site_config, owner=params.hive_user, group=params.user_group, mode=mode_identified) File(os.path.join(hive_server_interactive_conf_dir, "hadoop-metrics2-hiveserver2.properties"), owner=params.hive_user, group=params.user_group, mode=mode_identified, content=Template("hadoop-metrics2-hiveserver2.properties.j2")) File(format( "{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties" ), owner=params.hive_user, group=params.user_group, mode=mode_identified, content=Template("hadoop-metrics2-llapdaemon.j2")) File(format( "{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties" ), owner=params.hive_user, group=params.user_group, mode=mode_identified, content=Template("hadoop-metrics2-llaptaskscheduler.j2")) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2")) if not os.path.exists(params.target_hive_interactive): jdbc_connector(params.target_hive_interactive, params.hive_intaractive_previous_jdbc_jar) File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content=DownloadSource( format("{jdk_location}/{check_db_connection_jar_name}")), mode=0644) File(params.start_hiveserver2_interactive_path, mode=0755, content=Template(format('{start_hiveserver2_interactive_script}'))) Directory(params.hive_pid_dir, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) Directory(params.hive_log_dir, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) Directory(params.hive_interactive_var_lib, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) generate_logfeeder_input_config( 'hive', Template("input.config-hive.json.j2", extra_imports=[default]))
def setup_usersync(upgrade_type=None): import params usersync_home = params.usersync_home ranger_home = params.ranger_home ranger_ugsync_conf = params.ranger_ugsync_conf if not is_empty( params.ranger_usersync_ldap_ldapbindpassword ) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder': password_validation(params.ranger_usersync_ldap_ldapbindpassword) Directory(params.ranger_pid_dir, mode=0755, owner=params.unix_user, group=params.user_group, cd_access="a", create_parents=True) Directory(params.usersync_log_dir, owner=params.unix_user, group=params.unix_group, cd_access='a', create_parents=True, mode=0755, recursive_ownership=True) Directory(format("{ranger_ugsync_conf}/"), owner=params.unix_user) generate_logfeeder_input_config( 'ranger', Template("input.config-ranger.json.j2", extra_imports=[default])) if upgrade_type is not None: src_file = format( '{usersync_home}/conf.dist/ranger-ugsync-default.xml') dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) if params.stack_supports_ranger_log4j: File(format('{usersync_home}/conf/log4j.properties'), owner=params.unix_user, group=params.unix_group, content=InlineTemplate(params.usersync_log4j), mode=0644) elif upgrade_type is not None and not params.stack_supports_ranger_log4j: src_file = format('{usersync_home}/conf.dist/log4j.xml') dst_file = format('{usersync_home}/conf/log4j.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) # remove plain-text password from xml configs ranger_ugsync_site_copy = {} ranger_ugsync_site_copy.update( params.config['configurations']['ranger-ugsync-site']) for prop in params.ranger_usersync_password_properties: if prop in ranger_ugsync_site_copy: ranger_ugsync_site_copy[prop] = "_" XmlConfig("ranger-ugsync-site.xml", conf_dir=ranger_ugsync_conf, configurations=ranger_ugsync_site_copy, configuration_attributes=params.config['configurationAttributes'] ['ranger-ugsync-site'], owner=params.unix_user, group=params.unix_group, mode=0644) if os.path.isfile(params.ranger_ugsync_default_file): File(params.ranger_ugsync_default_file, owner=params.unix_user, group=params.unix_group) if os.path.isfile(params.usgsync_log4j_file): File(params.usgsync_log4j_file, owner=params.unix_user, group=params.unix_group) if os.path.isfile(params.cred_validator_file): File(params.cred_validator_file, group=params.unix_group, mode=04555) ranger_credential_helper(params.ugsync_cred_lib, 'usersync.ssl.key.password', params.ranger_usersync_keystore_password, params.ugsync_jceks_path) if not is_empty( params.ranger_usersync_ldap_ldapbindpassword ) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder': ranger_credential_helper(params.ugsync_cred_lib, 'ranger.usersync.ldap.bindalias', params.ranger_usersync_ldap_ldapbindpassword, params.ugsync_jceks_path) ranger_credential_helper(params.ugsync_cred_lib, 'usersync.ssl.truststore.password', params.ranger_usersync_truststore_password, params.ugsync_jceks_path) File(params.ugsync_jceks_path, owner=params.unix_user, group=params.unix_group, only_if=format("test -e {ugsync_jceks_path}"), mode=0640) update_dot_jceks_crc_ownership( credential_provider_path=params.ugsync_jceks_path, user=params.unix_user, group=params.unix_group) File( params.usersync_services_file, mode=0755, ) if not os.path.isfile(params.ranger_usersync_keystore_file): cmd = format( "{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'" ) Execute(cmd, logoutput=True, user=params.unix_user) File(params.ranger_usersync_keystore_file, owner=params.unix_user, group=params.unix_group, mode=0640) create_core_site_xml(ranger_ugsync_conf) File(format("{ranger_ugsync_conf}/ranger-usersync-env.sh"), content=InlineTemplate(params.ranger_env_content), owner=params.unix_user, group=params.unix_group, mode=0755)
def yarn(name=None, config_dir=None): """ :param name: Component name, apptimelinereader, apptimelineserver, nodemanager, resourcemanager, or None (defaults for client) :param config_dir: Which config directory to write configs to, which could be different during rolling upgrade. """ import params install_lzo_if_needed() if config_dir is None: config_dir = params.hadoop_conf_dir Directory( [params.yarn_log_dir_prefix], owner=params.yarn_user, group=params.user_group, create_parents=True, ignore_failures=True, cd_access='a', mode=0775, ) Directory( [params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir], owner=params.yarn_user, group=params.user_group, create_parents=True, cd_access='a', ) Directory( [ params.mapred_pid_dir_prefix, params.mapred_pid_dir, params.mapred_log_dir_prefix, params.mapred_log_dir ], owner=params.mapred_user, group=params.user_group, create_parents=True, cd_access='a', ) Directory( params.yarn_hbase_conf_dir, owner=params.yarn_hbase_user, group=params.user_group, create_parents=True, cd_access='a', ) # Some of these function calls depend on the directories above being created first. if name == 'resourcemanager': setup_resourcemanager() elif name == 'nodemanager': setup_nodemanager() elif name == 'apptimelineserver': setup_ats() elif name == 'historyserver': setup_historyserver() elif name == 'apptimelinereader': if not params.is_hbase_system_service_launch: setup_atsv2_hbase_directories() setup_atsv2_hbase_files() generate_logfeeder_input_config( 'yarn', Template("input.config-yarn.json.j2", extra_imports=[default])) # if there is the viewFS mount table content, create separate xml config and include in in the core-site # else just create core-site if params.mount_table_content: XmlConfig( "core-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['core-site'], configuration_attributes=params.config['configurationAttributes'] ['core-site'], owner=params.hdfs_user, group=params.user_group, mode=0644, xml_include_file=os.path.join(config_dir, params.xml_inclusion_file_name)) File(os.path.join(config_dir, params.xml_inclusion_file_name), owner=params.hdfs_user, group=params.user_group, content=params.mount_table_content, mode=0644) else: XmlConfig( "core-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['core-site'], configuration_attributes=params.config['configurationAttributes'] ['core-site'], owner=params.hdfs_user, group=params.user_group, mode=0644) # During RU, Core Masters and Slaves need hdfs-site.xml # TODO, instead of specifying individual configs, which is susceptible to breaking when new configs are added, # RU should rely on all available in <stack-root>/<version>/hadoop/conf XmlConfig("hdfs-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params.config['configurationAttributes'] ['hdfs-site'], owner=params.hdfs_user, group=params.user_group, mode=0644) XmlConfig("mapred-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['mapred-site'], configuration_attributes=params.config['configurationAttributes'] ['mapred-site'], owner=params.yarn_user, group=params.user_group, mode=0644) configs = {} configs.update(params.config['configurations']['yarn-site']) configs["hadoop.registry.dns.bind-port"] = params.config['configurations'][ 'yarn-env']['registry.dns.bind-port'] XmlConfig("yarn-site.xml", conf_dir=config_dir, configurations=configs, configuration_attributes=params.config['configurationAttributes'] ['yarn-site'], owner=params.yarn_user, group=params.user_group, mode=0644) XmlConfig( "capacity-scheduler.xml", conf_dir=config_dir, configurations=params.config['configurations']['capacity-scheduler'], configuration_attributes=params.config['configurationAttributes'] ['capacity-scheduler'], owner=params.yarn_user, group=params.user_group, mode=0644) XmlConfig( "hbase-site.xml", conf_dir=params.yarn_hbase_conf_dir, configurations=params.config['configurations']['yarn-hbase-site'], configuration_attributes=params.config['configurationAttributes'] ['yarn-hbase-site'], owner=params.yarn_hbase_user, group=params.user_group, mode=0644) XmlConfig("resource-types.xml", conf_dir=config_dir, configurations=params.config['configurations']['resource-types'], configuration_attributes=params.config['configurationAttributes'] ['resource-types'], owner=params.yarn_user, group=params.user_group, mode=0644) File(format("{limits_conf_dir}/yarn.conf"), mode=0644, content=Template('yarn.conf.j2')) File(format("{limits_conf_dir}/mapreduce.conf"), mode=0644, content=Template('mapreduce.conf.j2')) File(os.path.join(config_dir, "yarn-env.sh"), owner=params.yarn_user, group=params.user_group, mode=0755, content=InlineTemplate(params.yarn_env_sh_template)) File(format("{yarn_bin}/container-executor"), group=params.yarn_executor_container_group, mode=params.container_executor_mode) File(os.path.join(config_dir, "container-executor.cfg"), group=params.user_group, mode=0644, content=InlineTemplate(params.container_executor_cfg_template)) Directory(params.cgroups_dir, group=params.user_group, create_parents=True, mode=0755, cd_access="a") File(os.path.join(config_dir, "mapred-env.sh"), owner=params.tc_owner, mode=0755, content=InlineTemplate(params.mapred_env_sh_template)) if params.yarn_nodemanager_recovery_dir: Directory( InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(), owner=params.yarn_user, group=params.user_group, create_parents=True, mode=0755, cd_access='a', ) if params.security_enabled: File(os.path.join(params.hadoop_bin, "task-controller"), owner="root", group=params.mapred_tt_group, mode=06050) File(os.path.join(config_dir, 'taskcontroller.cfg'), owner=params.tc_owner, mode=params.tc_mode, group=params.mapred_tt_group, content=Template("taskcontroller.cfg.j2")) File(os.path.join(config_dir, 'yarn_jaas.conf'), owner=params.yarn_user, group=params.user_group, content=Template("yarn_jaas.conf.j2")) if params.has_ats: File(os.path.join(config_dir, 'yarn_ats_jaas.conf'), owner=params.yarn_user, group=params.user_group, content=Template("yarn_ats_jaas.conf.j2")) if params.has_registry_dns: File(os.path.join(config_dir, 'yarn_registry_dns_jaas.conf'), owner=params.yarn_user, group=params.user_group, content=Template("yarn_registry_dns_jaas.conf.j2")) File(os.path.join(config_dir, 'yarn_nm_jaas.conf'), owner=params.yarn_user, group=params.user_group, content=Template("yarn_nm_jaas.conf.j2")) if params.has_hs: File(os.path.join(config_dir, 'mapred_jaas.conf'), owner=params.mapred_user, group=params.user_group, content=Template("mapred_jaas.conf.j2")) else: File(os.path.join(config_dir, 'taskcontroller.cfg'), owner=params.tc_owner, content=Template("taskcontroller.cfg.j2")) XmlConfig("mapred-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['mapred-site'], configuration_attributes=params.config['configurationAttributes'] ['mapred-site'], owner=params.mapred_user, group=params.user_group) XmlConfig( "capacity-scheduler.xml", conf_dir=config_dir, configurations=params.config['configurations']['capacity-scheduler'], configuration_attributes=params.config['configurationAttributes'] ['capacity-scheduler'], owner=params.hdfs_user, group=params.user_group) if "ssl-client" in params.config['configurations']: XmlConfig( "ssl-client.xml", conf_dir=config_dir, configurations=params.config['configurations']['ssl-client'], configuration_attributes=params.config['configurationAttributes'] ['ssl-client'], owner=params.hdfs_user, group=params.user_group) Directory( params.hadoop_conf_secure_dir, create_parents=True, owner='root', group=params.user_group, cd_access='a', ) XmlConfig( "ssl-client.xml", conf_dir=params.hadoop_conf_secure_dir, configurations=params.config['configurations']['ssl-client'], configuration_attributes=params.config['configurationAttributes'] ['ssl-client'], owner=params.hdfs_user, group=params.user_group) if "ssl-server" in params.config['configurations']: XmlConfig( "ssl-server.xml", conf_dir=config_dir, configurations=params.config['configurations']['ssl-server'], configuration_attributes=params.config['configurationAttributes'] ['ssl-server'], owner=params.hdfs_user, group=params.user_group) if os.path.exists(os.path.join(config_dir, 'fair-scheduler.xml')): File(os.path.join(config_dir, 'fair-scheduler.xml'), owner=params.mapred_user, group=params.user_group) if os.path.exists(os.path.join(config_dir, 'ssl-client.xml.example')): File(os.path.join(config_dir, 'ssl-client.xml.example'), owner=params.mapred_user, group=params.user_group) if os.path.exists(os.path.join(config_dir, 'ssl-server.xml.example')): File(os.path.join(config_dir, 'ssl-server.xml.example'), owner=params.mapred_user, group=params.user_group) setup_atsv2_backend(name, config_dir)
def oozie(is_server=False, upgrade_type=None): import params if is_server: params.HdfsResource(params.oozie_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.oozie_user, mode=params.oozie_hdfs_user_mode) params.HdfsResource(None, action="execute") generate_logfeeder_input_config( 'oozie', Template("input.config-oozie.json.j2", extra_imports=[default])) Directory(params.conf_dir, create_parents=True, owner=params.oozie_user, group=params.user_group) params.oozie_site = update_credential_provider_path( params.oozie_site, 'oozie-site', os.path.join(params.conf_dir, 'oozie-site.jceks'), params.oozie_user, params.user_group, use_local_jceks=True) XmlConfig("oozie-site.xml", conf_dir=params.conf_dir, configurations=params.oozie_site, configuration_attributes=params.config['configurationAttributes'] ['oozie-site'], owner=params.oozie_user, group=params.user_group, mode=0664) File( format("{conf_dir}/oozie-env.sh"), owner=params.oozie_user, content=InlineTemplate(params.oozie_env_sh_template), group=params.user_group, ) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'oozie.conf'), owner='root', group='root', mode=0644, content=Template("oozie.conf.j2")) if (params.log4j_props != None): File(format("{params.conf_dir}/oozie-log4j.properties"), mode=0644, group=params.user_group, owner=params.oozie_user, content=InlineTemplate(params.log4j_props)) elif (os.path.exists(format("{params.conf_dir}/oozie-log4j.properties"))): File(format("{params.conf_dir}/oozie-log4j.properties"), mode=0644, group=params.user_group, owner=params.oozie_user) if params.stack_version_formatted and check_stack_feature( StackFeature.OOZIE_ADMIN_USER, params.stack_version_formatted): File(format("{params.conf_dir}/adminusers.txt"), mode=0644, group=params.user_group, owner=params.oozie_user, content=Template('adminusers.txt.j2', oozie_admin_users=params.oozie_admin_users)) else: File(format("{params.conf_dir}/adminusers.txt"), owner=params.oozie_user, group=params.user_group) if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \ params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \ params.jdbc_driver_name == "org.postgresql.Driver" or \ params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver": File( format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content=DownloadSource( format("{jdk_location}/{check_db_connection_jar_name}")), ) pass oozie_ownership() if params.lzo_enabled: install_lzo_if_needed() Execute( format( '{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), ) if is_server: oozie_server_specific(upgrade_type)
def druid(upgrade_type=None, nodeType=None): import params ensure_base_directories() # Environment Variables File(format("{params.druid_conf_dir}/druid-env.sh"), owner=params.druid_user, content=InlineTemplate(params.druid_env_sh_template), mode=0700) # common config druid_common_config = mutable_config_dict( params.config['configurations']['druid-common']) # User cannot override below configs druid_common_config['druid.host'] = params.hostname druid_common_config[ 'druid.extensions.directory'] = params.druid_extensions_dir druid_common_config[ 'druid.extensions.hadoopDependenciesDir'] = params.druid_hadoop_dependencies_dir druid_common_config[ 'druid.selectors.indexing.serviceName'] = params.config[ 'configurations']['druid-overlord']['druid.service'] druid_common_config['druid.selectors.coordinator.serviceName'] = \ params.config['configurations']['druid-coordinator']['druid.service'] druid_common_config['druid.extensions.loadList'] = json.dumps( eval(params.druid_extensions_load_list) + eval(params.druid_security_extensions_load_list)) # delete the password and user if empty otherwiswe derby will fail. if 'derby' == druid_common_config['druid.metadata.storage.type']: del druid_common_config['druid.metadata.storage.connector.user'] del druid_common_config['druid.metadata.storage.connector.password'] druid_env_config = mutable_config_dict( params.config['configurations']['druid-env']) PropertiesFile("common.runtime.properties", dir=params.druid_common_conf_dir, properties=druid_common_config, owner=params.druid_user, group=params.user_group, mode=0600) Logger.info("Created common.runtime.properties") File(format("{params.druid_common_conf_dir}/druid-log4j.xml"), mode=0644, owner=params.druid_user, group=params.user_group, content=InlineTemplate(params.log4j_props)) Logger.info("Created log4j file") File("/etc/logrotate.d/druid", mode=0644, owner='root', group='root', content=InlineTemplate(params.logrotate_props)) Logger.info("Created log rotate file") if params.security_enabled: TemplateConfig(params.druid_jaas_file, owner=params.druid_user, group=params.user_group, mode=0644) # node specific configs for node_type in [ 'coordinator', 'overlord', 'historical', 'broker', 'middleManager', 'router' ]: node_config_dir = format('{params.druid_conf_dir}/{node_type}') node_type_lowercase = node_type.lower() # Write runtime.properties file node_config = mutable_config_dict(params.config['configurations'][ format('druid-{node_type_lowercase}')]) if (node_type == 'middleManager'): # Replace correct values for stack_version and druid_jaas_file node_config['druid.indexer.runner.javaOpts'] = format( node_config['druid.indexer.runner.javaOpts']) PropertiesFile("runtime.properties", dir=node_config_dir, properties=node_config, owner=params.druid_user, group=params.user_group, mode=0600) Logger.info( format("Created druid-{node_type_lowercase} runtime.properties")) # Write jvm configs File( format('{node_config_dir}/jvm.config'), owner=params.druid_user, group=params.user_group, content=InlineTemplate( "-server \n-Xms{{node_heap_memory}}m \n-Xmx{{node_heap_memory}}m \n-XX:MaxDirectMemorySize={{node_direct_memory}}m \n-Dlog4j.configurationFile={{log4j_config_file}} \n-Dlog4j.debug \n{{node_jvm_opts}}", node_heap_memory=druid_env_config[format( 'druid.{node_type_lowercase}.jvm.heap.memory')], log4j_config_file=format( "{params.druid_common_conf_dir}/druid-log4j.xml"), node_direct_memory=druid_env_config[format( 'druid.{node_type_lowercase}.jvm.direct.memory')], node_jvm_opts=format(druid_env_config[format( 'druid.{node_type_lowercase}.jvm.opts')]))) Logger.info(format("Created druid-{node_type_lowercase} jvm.config")) # Handling hadoop Lzo jars if enable and node type is hadoop related eg Overlords and MMs if ['middleManager', 'overlord' ].__contains__(node_type_lowercase) and params.lzo_enabled: try: Logger.info( format( "Copying hadoop lzo jars from {hadoop_lib_home} to {druid_hadoop_dependencies_dir}/hadoop-client/*/" )) Execute( format( '{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {druid_hadoop_dependencies_dir}/hadoop-client/*/' )) except Fail as ex: Logger.info( format( "No Hadoop LZO found at {hadoop_lib_home}/hadoop-lzo*.jar" )) generate_logfeeder_input_config( 'druid', Template("input.config-druid.json.j2", extra_imports=[default])) # All druid nodes have dependency on hdfs_client ensure_hadoop_directories() download_database_connector_if_needed() # Pull all required dependencies pulldeps()
def flume(action=None): import params if action == 'config': if params.security_enabled: File( format("{flume_conf_dir}/flume_jaas.conf"), owner=params.flume_user, content=InlineTemplate(params.flume_jaas_conf_template)) Directory([params.flume_log_dir, params.flume_conf_dir, params.flume_run_dir], owner=params.flume_user, group=params.flume_group, mode=0775, cd_access="a", create_parents=True ) File(params.flume_conf_dir + '/flume.conf', content=InlineTemplate(params.flume_conf_content), owner=params.flume_user, mode=0644) File( params.flume_conf_dir + '/log4j.properties', content=Template('log4j.properties.j2'), owner=params.flume_user, mode=0644) File( params.flume_conf_dir + '/flume-env.sh', owner=params.flume_user, content=InlineTemplate(params.flume_env_sh_template)) if params.has_metric_collector: File(params.flume_conf_dir + '/flume-metrics2.properties', owner=params.flume_user, content=Template("flume-metrics2.properties.j2")) generate_logfeeder_input_config( 'flume', Template("input.config-flume.json.j2", extra_imports=[default])) elif action == 'start': flume_base = as_user( format( "{flume_bin} agent --name {{0}} --conf {{1}} --conf-file {{2}} {{3}} > {flume_log_dir}/flume.out 2>&1" ), params.flume_user, env={'JAVA_HOME': params.java_home}) + " &" extra_args = '' if params.has_metric_collector: extra_args = '-Dflume.monitoring.type=org.apache.hadoop.metrics2.sink.flume.FlumeTimelineMetricsSink ' \ '-Dflume.monitoring.node={0}:{1}' extra_args = extra_args.format( params.metric_collector_host, params.metric_collector_port) flume_cmd = flume_base.format('ingest', params.flume_conf_dir, params.flume_conf_dir + '/flume.conf', extra_args) Execute( flume_cmd, wait_for_finish=False, environment={'JAVA_HOME': params.java_home}) pid_cmd = as_sudo(('pgrep', '-o', '-u', params.flume_user, '-f', format('^{java_home}'))) + \ " | " + as_sudo(('tee', params.flume_pid_file)) + " && test ${PIPESTATUS[0]} -eq 0" try: Execute(pid_cmd, logoutput=True, tries=20, try_sleep=10) except: show_logs(params.flume_log_dir, params.flume_user) elif action == 'stop': pid = shell.checked_call(("cat", params.flume_pid_file), sudo=True)[1].strip() Execute(("kill", "-15", pid), sudo=True) show_logs(params.flume_log_dir, params.flume_user) File(params.flume_pid_file, action='delete')
def metadata(type='server'): import params # Needed by both Server and Client Directory(params.conf_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) if type == "server": Directory([params.pid_dir], mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(format('{conf_dir}/solr'), mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True, recursive_ownership=True) Directory(params.log_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.data_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.expanded_war_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) File(format("{expanded_war_dir}/atlas.war"), content=StaticFile( format('{metadata_home}/server/webapp/atlas.war'))) File(format("{conf_dir}/atlas-log4j.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_log4j_content)) File(format("{conf_dir}/atlas-env.sh"), owner=params.metadata_user, group=params.user_group, mode=0755, content=InlineTemplate(params.metadata_env_content)) if not is_empty(params.atlas_admin_username) and not is_empty( params.atlas_admin_password): psswd_output = hashlib.sha256( params.atlas_admin_password).hexdigest() ModifyPropertiesFile( format("{conf_dir}/users-credentials.properties"), properties={ format('{atlas_admin_username}'): format('ROLE_ADMIN::{psswd_output}') }, owner=params.metadata_user) files_to_chown = [ format("{conf_dir}/atlas-simple-authz-policy.json"), format("{conf_dir}/users-credentials.properties") ] for file in files_to_chown: if os.path.exists(file): Execute( ('chown', format('{metadata_user}:{user_group}'), file), sudo=True) Execute(('chmod', '640', file), sudo=True) if params.metadata_solrconfig_content: File(format("{conf_dir}/solr/solrconfig.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_solrconfig_content)) generate_logfeeder_input_config( 'atlas', Template("input.config-atlas.json.j2", extra_imports=[default])) # Needed by both Server and Client PropertiesFile(format('{conf_dir}/{conf_file}'), properties=params.application_properties, mode=0600, owner=params.metadata_user, group=params.user_group) if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user) if type == 'server' and params.search_backend_solr and params.has_infra_solr: solr_cloud_util.setup_solr_client(params.config) check_znode() jaasFile = params.atlas_jaas_file if params.security_enabled else None upload_conf_set('atlas_configs', jaasFile) if params.security_enabled: # update permissions before creating the collections solr_cloud_util.add_solr_roles( params.config, roles=[ params.infra_solr_role_atlas, params.infra_solr_role_ranger_audit, params.infra_solr_role_dev ], new_service_principals=[params.atlas_jaas_principal]) create_collection('vertex_index', 'atlas_configs', jaasFile) create_collection('edge_index', 'atlas_configs', jaasFile) create_collection('fulltext_index', 'atlas_configs', jaasFile) if params.security_enabled: secure_znode(format('{infra_solr_znode}/configs/atlas_configs'), jaasFile) secure_znode(format('{infra_solr_znode}/collections/vertex_index'), jaasFile) secure_znode(format('{infra_solr_znode}/collections/edge_index'), jaasFile) secure_znode( format('{infra_solr_znode}/collections/fulltext_index'), jaasFile) File(params.atlas_hbase_setup, group=params.user_group, owner=params.hbase_user, content=Template("atlas_hbase_setup.rb.j2")) is_atlas_upgrade_support = check_stack_feature( StackFeature.ATLAS_UPGRADE_SUPPORT, get_stack_feature_version(params.config)) if is_atlas_upgrade_support and params.security_enabled: File(params.atlas_kafka_setup, group=params.user_group, owner=params.kafka_user, content=Template("atlas_kafka_acl.sh.j2")) # files required only in case if kafka broker is not present on the host as configured component if not params.host_with_kafka: File(format("{kafka_conf_dir}/kafka-env.sh"), owner=params.kafka_user, content=InlineTemplate(params.kafka_env_sh_template)) File(format("{kafka_conf_dir}/kafka_jaas.conf"), group=params.user_group, owner=params.kafka_user, content=Template("kafka_jaas.conf.j2")) if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len( params.namenode_host) > 1: XmlConfig( "hdfs-site.xml", conf_dir=params.conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params.config['configurationAttributes'] ['hdfs-site'], owner=params.metadata_user, group=params.user_group, mode=0644) else: File(format('{conf_dir}/hdfs-site.xml'), action="delete") ''' Atlas requires hadoop core-site.xml to resolve users/groups synced in HadoopUGI for authentication and authorization process. Earlier the core-site.xml was available in Hbase conf directory which is a part of Atlas class-path, from stack 2.6 onwards, core-site.xml is no more available in Hbase conf directory. Hence need to create core-site.xml in Atlas conf directory. ''' if params.stack_supports_atlas_core_site and params.has_namenode: XmlConfig( "core-site.xml", conf_dir=params.conf_dir, configurations=params.config['configurations']['core-site'], configuration_attributes=params.config['configurationAttributes'] ['core-site'], owner=params.metadata_user, group=params.user_group, mode=0644, xml_include_file=params.mount_table_xml_inclusion_file_full_path) if params.mount_table_content: File(params.mount_table_xml_inclusion_file_full_path, owner=params.metadata_user, group=params.user_group, content=params.mount_table_content, mode=0644) Directory( format('{metadata_home}/'), owner=params.metadata_user, group=params.user_group, recursive_ownership=True, )
def setup_spark(env, type, upgrade_type = None, action = None): import params # ensure that matching LZO libraries are installed for Spark lzo_utils.install_lzo_if_needed() Directory([params.spark_pid_dir, params.spark_log_dir], owner=params.spark_user, group=params.user_group, mode=0775, create_parents = True ) if type == 'server' and action == 'config': params.HdfsResource(params.spark_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.spark_user, mode=0775 ) if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(params.default_fs).scheme: # Create Spark Warehouse Dir params.HdfsResource(params.spark_warehouse_dir, type="directory", action="create_on_execute", owner=params.spark_user, mode=0777 ) params.HdfsResource(None, action="execute") generate_logfeeder_input_config('spark2', Template("input.config-spark2.json.j2", extra_imports=[default])) spark2_defaults = dict(params.config['configurations']['spark2-defaults']) if params.security_enabled: spark2_defaults.pop("history.server.spnego.kerberos.principal") spark2_defaults.pop("history.server.spnego.keytab.file") spark2_defaults['spark.history.kerberos.principal'] = spark2_defaults['spark.history.kerberos.principal'].replace('_HOST', socket.getfqdn().lower()) PropertiesFile(format("{spark_conf}/spark-defaults.conf"), properties = spark2_defaults, key_value_delimiter = " ", owner=params.spark_user, group=params.spark_group, mode=0644 ) # create spark-env.sh in etc/conf dir File(os.path.join(params.spark_conf, 'spark-env.sh'), owner=params.spark_user, group=params.spark_group, content=InlineTemplate(params.spark_env_sh), mode=0644, ) #create log4j.properties in etc/conf dir File(os.path.join(params.spark_conf, 'log4j.properties'), owner=params.spark_user, group=params.spark_group, content=params.spark_log4j_properties, mode=0644, ) #create metrics.properties in etc/conf dir File(os.path.join(params.spark_conf, 'metrics.properties'), owner=params.spark_user, group=params.spark_group, content=InlineTemplate(params.spark_metrics_properties), mode=0644 ) if params.is_hive_installed: XmlConfig("hive-site.xml", conf_dir=params.spark_conf, configurations=params.spark_hive_properties, owner=params.spark_user, group=params.spark_group, mode=0644) if params.has_spark_thriftserver: spark2_thrift_sparkconf = dict(params.config['configurations']['spark2-thrift-sparkconf']) if params.security_enabled and 'spark.yarn.principal' in spark2_thrift_sparkconf: spark2_thrift_sparkconf['spark.yarn.principal'] = spark2_thrift_sparkconf['spark.yarn.principal'].replace('_HOST', socket.getfqdn().lower()) PropertiesFile(params.spark_thrift_server_conf_file, properties = spark2_thrift_sparkconf, owner = params.hive_user, group = params.user_group, key_value_delimiter = " ", mode=0644 ) effective_version = params.version if upgrade_type is not None else params.stack_version_formatted if effective_version: effective_version = format_stack_version(effective_version) if params.spark_thrift_fairscheduler_content and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version): # create spark-thrift-fairscheduler.xml File(os.path.join(params.spark_conf,"spark-thrift-fairscheduler.xml"), owner=params.spark_user, group=params.spark_group, mode=0755, content=InlineTemplate(params.spark_thrift_fairscheduler_content) )
def ams(name=None, action=None): import params if name == 'collector': Directory( params.ams_collector_conf_dir, owner=params.ams_user, group=params.user_group, create_parents=True, recursive_ownership=True, ) Directory(params.ams_checkpoint_dir, owner=params.ams_user, group=params.user_group, cd_access="a", create_parents=True, recursive_ownership=True) new_ams_site = {} new_ams_site.update(params.config['configurations']['ams-site']) if params.clusterHostInfoDict: master_components = [] slave_components = [] components = dict(params.clusterHostInfoDict).keys() known_slave_components = [ "nodemanager", "metrics_monitor", "datanode", "hbase_regionserver" ] for component in components: if component and component.endswith("_hosts"): component_name = component[:-6] elif component and component.endswith("_host"): component_name = component[:-5] else: continue if component_name in known_slave_components: slave_components.append(component_name) else: master_components.append(component_name) if slave_components: new_ams_site[ 'timeline.metrics.initial.configured.slave.components'] = ",".join( slave_components) if master_components: if 'ambari_server' not in master_components: master_components.append('ambari_server') new_ams_site[ 'timeline.metrics.initial.configured.master.components'] = ",".join( master_components) hbase_total_heapsize_with_trailing_m = params.hbase_heapsize hbase_total_heapsize = int( hbase_total_heapsize_with_trailing_m[:-1]) * 1024 * 1024 new_ams_site['hbase_total_heapsize'] = hbase_total_heapsize XmlConfig( "ams-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=new_ams_site, configuration_attributes=params.config['configurationAttributes'] ['ams-site'], owner=params.ams_user, group=params.user_group) XmlConfig( "ssl-server.xml", conf_dir=params.ams_collector_conf_dir, configurations=params.config['configurations']['ams-ssl-server'], configuration_attributes=params.config['configurationAttributes'] ['ams-ssl-server'], owner=params.ams_user, group=params.user_group) merged_ams_hbase_site = {} merged_ams_hbase_site.update( params.config['configurations']['ams-hbase-site']) if params.security_enabled: merged_ams_hbase_site.update( params.config['configurations']['ams-hbase-security-site']) # Add phoenix client side overrides merged_ams_hbase_site['phoenix.query.maxGlobalMemoryPercentage'] = str( params.phoenix_max_global_mem_percent) merged_ams_hbase_site[ 'phoenix.spool.directory'] = params.phoenix_client_spool_dir XmlConfig( "hbase-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=merged_ams_hbase_site, configuration_attributes=params.config['configurationAttributes'] ['ams-hbase-site'], owner=params.ams_user, group=params.user_group) if params.security_enabled: TemplateConfig(os.path.join(params.hbase_conf_dir, "ams_collector_jaas.conf"), owner=params.ams_user, template_tag=None) if (params.log4j_props != None): File(format("{params.ams_collector_conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.ams_user, content=InlineTemplate(params.log4j_props)) File(format("{ams_collector_conf_dir}/ams-env.sh"), owner=params.ams_user, content=InlineTemplate(params.ams_env_sh_template)) Directory( params.ams_collector_log_dir, owner=params.ams_user, group=params.user_group, cd_access="a", create_parents=True, mode=0755, ) Directory( params.ams_collector_pid_dir, owner=params.ams_user, group=params.user_group, cd_access="a", create_parents=True, mode=0755, ) # Hack to allow native HBase libs to be included for embedded hbase File(os.path.join(params.ams_hbase_home_dir, "bin", "hadoop"), owner=params.ams_user, mode=0755) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') # Setting up security limits File(os.path.join(params.limits_conf_dir, 'ams.conf'), owner='root', group='root', mode=0644, content=Template("ams.conf.j2")) # Phoenix spool file dir if not /tmp if not os.path.exists(params.phoenix_client_spool_dir): Directory(params.phoenix_client_spool_dir, owner=params.ams_user, mode=0755, group=params.user_group, cd_access="a", create_parents=True) pass if not params.is_local_fs_rootdir and params.is_ams_distributed: # Configuration needed to support NN HA XmlConfig( "hdfs-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params. config['configurationAttributes']['hdfs-site'], owner=params.ams_user, group=params.user_group, mode=0644) XmlConfig( "hdfs-site.xml", conf_dir=params.hbase_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params. config['configurationAttributes']['hdfs-site'], owner=params.ams_user, group=params.user_group, mode=0644) # Remove spnego configs from core-site if platform does not have python-kerberos library truncated_core_site = {} truncated_core_site.update( params.config['configurations']['core-site']) if is_spnego_enabled(params): truncated_core_site.pop('hadoop.http.authentication.type') truncated_core_site.pop('hadoop.http.filter.initializers') # if there is the viewFS mount table content, create separate xml config and include in in the core-sites # else just create core-sites if params.mount_table_content: XmlConfig("core-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=truncated_core_site, configuration_attributes=params. config['configurationAttributes']['core-site'], owner=params.ams_user, group=params.user_group, mode=0644, xml_include_file=os.path.join( params.ams_collector_conf_dir, params.xml_inclusion_file_name)) File(os.path.join(params.ams_collector_conf_dir, params.xml_inclusion_file_name), owner=params.ams_user, group=params.user_group, content=params.mount_table_content, mode=0644) XmlConfig("core-site.xml", conf_dir=params.hbase_conf_dir, configurations=truncated_core_site, configuration_attributes=params. config['configurationAttributes']['core-site'], owner=params.ams_user, group=params.user_group, mode=0644, xml_include_file=os.path.join( params.hbase_conf_dir, params.xml_inclusion_file_name)) File(os.path.join(params.hbase_conf_dir, params.xml_inclusion_file_name), owner=params.ams_user, group=params.user_group, content=params.mount_table_content, mode=0644) else: XmlConfig("core-site.xml", conf_dir=params.ams_collector_conf_dir, configurations=truncated_core_site, configuration_attributes=params. config['configurationAttributes']['core-site'], owner=params.ams_user, group=params.user_group, mode=0644) XmlConfig("core-site.xml", conf_dir=params.hbase_conf_dir, configurations=truncated_core_site, configuration_attributes=params. config['configurationAttributes']['core-site'], owner=params.ams_user, group=params.user_group, mode=0644) if params.metric_collector_https_enabled: export_ca_certs(params.ams_collector_conf_dir) pass elif name == 'monitor': # TODO Uncomment when SPNEGO support has been added to AMS service check and Grafana. if is_spnego_enabled(params) and is_redhat_centos_6_plus(): try: import kerberos except ImportError: raise ImportError( "python-kerberos package need to be installed to run AMS in SPNEGO mode" ) Directory(params.ams_monitor_conf_dir, owner=params.ams_user, group=params.user_group, create_parents=True) Directory(params.ams_monitor_log_dir, owner=params.ams_user, group=params.user_group, mode=0755, create_parents=True) if params.host_in_memory_aggregation and params.log4j_props is not None: File(format("{params.ams_monitor_conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.ams_user, content=InlineTemplate(params.log4j_props)) XmlConfig( "ams-site.xml", conf_dir=params.ams_monitor_conf_dir, configurations=params.config['configurations']['ams-site'], configuration_attributes=params. config['configurationAttributes']['ams-site'], owner=params.ams_user, group=params.user_group) XmlConfig("ssl-server.xml", conf_dir=params.ams_monitor_conf_dir, configurations=params.config['configurations'] ['ams-ssl-server'], configuration_attributes=params. config['configurationAttributes']['ams-ssl-server'], owner=params.ams_user, group=params.user_group) pass Execute( format( "{sudo} chown -R {ams_user}:{user_group} {ams_monitor_log_dir}" )) Directory(params.ams_monitor_pid_dir, owner=params.ams_user, group=params.user_group, cd_access="a", mode=0755, create_parents=True) Directory(format("{ams_monitor_dir}/psutil/build"), owner=params.ams_user, group=params.user_group, cd_access="a", create_parents=True) Execute( format( "{sudo} chown -R {ams_user}:{user_group} {ams_monitor_dir}")) TemplateConfig(format("{ams_monitor_conf_dir}/metric_monitor.ini"), owner=params.ams_user, group=params.user_group, template_tag=None) TemplateConfig(format("{ams_monitor_conf_dir}/metric_groups.conf"), owner=params.ams_user, group=params.user_group, template_tag=None) File(format("{ams_monitor_conf_dir}/ams-env.sh"), owner=params.ams_user, content=InlineTemplate(params.ams_env_sh_template)) if params.metric_collector_https_enabled or params.is_aggregation_https_enabled: export_ca_certs(params.ams_monitor_conf_dir) pass elif name == 'grafana': ams_grafana_directories = [ params.ams_grafana_conf_dir, params.ams_grafana_log_dir, params.ams_grafana_data_dir, params.ams_grafana_pid_dir ] for ams_grafana_directory in ams_grafana_directories: Directory(ams_grafana_directory, owner=params.ams_user, group=params.user_group, mode=0755, create_parents=True, recursive_ownership=True) File(format("{ams_grafana_conf_dir}/ams-grafana-env.sh"), owner=params.ams_user, group=params.user_group, content=InlineTemplate(params.ams_grafana_env_sh_template)) File(format("{ams_grafana_conf_dir}/ams-grafana.ini"), owner=params.ams_user, group=params.user_group, content=InlineTemplate(params.ams_grafana_ini_template), mode=0600) if action != 'stop': for dir in ams_grafana_directories: Execute(('chown', '-R', params.ams_user, dir), sudo=True) if params.metric_collector_https_enabled: export_ca_certs(params.ams_grafana_conf_dir) pass generate_logfeeder_input_config( 'ambari-metrics', Template("input.config-ambari-metrics.json.j2", extra_imports=[default]))
def kafka(upgrade_type=None): import params ensure_base_directories() kafka_server_config = mutable_config_dict( params.config['configurations']['kafka-broker']) # This still has an issue of hostnames being alphabetically out-of-order for broker.id in HDP-2.2. # Starting in HDP 2.3, Kafka handles the generation of broker.id so Ambari doesn't have to. effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version( params.version) Logger.info(format("Effective stack version: {effective_version}")) # listeners and advertised.listeners are only added in 2.3.0.0 onwards. if effective_version is not None and effective_version != "" and \ check_stack_feature(StackFeature.KAFKA_LISTENERS, effective_version): listeners = kafka_server_config['listeners'].replace( "localhost", params.hostname) Logger.info(format("Kafka listeners: {listeners}")) kafka_server_config['listeners'] = listeners if params.kerberos_security_enabled and params.kafka_kerberos_enabled: Logger.info("Kafka kerberos security is enabled.") if "SASL" not in listeners: listeners = kafka_server_config['listeners'] listeners = re.sub(r"(^|\b)PLAINTEXT://", "SASL_PLAINTEXT://", listeners) listeners = re.sub(r"(^|\b)PLAINTEXTSASL://", "SASL_PLAINTEXT://", listeners) listeners = re.sub(r"(^|\b)SSL://", "SASL_SSL://", listeners) kafka_server_config['listeners'] = listeners kafka_server_config['advertised.listeners'] = listeners Logger.info(format("Kafka advertised listeners: {listeners}")) elif 'advertised.listeners' in kafka_server_config: advertised_listeners = kafka_server_config[ 'advertised.listeners'].replace("localhost", params.hostname) kafka_server_config['advertised.listeners'] = advertised_listeners Logger.info( format("Kafka advertised listeners: {advertised_listeners}")) else: kafka_server_config['host.name'] = params.hostname if params.has_metric_collector: kafka_server_config[ 'kafka.timeline.metrics.hosts'] = params.ams_collector_hosts kafka_server_config[ 'kafka.timeline.metrics.port'] = params.metric_collector_port kafka_server_config[ 'kafka.timeline.metrics.protocol'] = params.metric_collector_protocol kafka_server_config[ 'kafka.timeline.metrics.truststore.path'] = params.metric_truststore_path kafka_server_config[ 'kafka.timeline.metrics.truststore.type'] = params.metric_truststore_type kafka_server_config[ 'kafka.timeline.metrics.truststore.password'] = params.metric_truststore_password kafka_data_dir = kafka_server_config['log.dirs'] kafka_data_dirs = filter(None, kafka_data_dir.split(",")) rack = "/default-rack" i = 0 if len(params.all_racks) > 0: for host in params.all_hosts: if host == params.hostname: rack = params.all_racks[i] break i = i + 1 Directory( kafka_data_dirs, mode=0755, cd_access='a', owner=params.kafka_user, group=params.user_group, create_parents=True, recursive_ownership=True, ) PropertiesFile( "server.properties", mode=0640, dir=params.conf_dir, properties=kafka_server_config, owner=params.kafka_user, group=params.user_group, ) File(format("{conf_dir}/kafka-env.sh"), owner=params.kafka_user, content=InlineTemplate(params.kafka_env_sh_template)) if (params.log4j_props != None): File(format("{conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.kafka_user, content=InlineTemplate(params.log4j_props)) if (params.kerberos_security_enabled and params.kafka_kerberos_enabled) or params.kafka_other_sasl_enabled: if params.kafka_jaas_conf_template: File(format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user, content=InlineTemplate(params.kafka_jaas_conf_template)) else: TemplateConfig(format("{conf_dir}/kafka_jaas.conf"), owner=params.kafka_user) if params.kafka_client_jaas_conf_template: File(format("{conf_dir}/kafka_client_jaas.conf"), owner=params.kafka_user, content=InlineTemplate( params.kafka_client_jaas_conf_template)) else: TemplateConfig(format("{conf_dir}/kafka_client_jaas.conf"), owner=params.kafka_user) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'kafka.conf'), owner='root', group='root', mode=0644, content=Template("kafka.conf.j2")) File(os.path.join(params.conf_dir, 'tools-log4j.properties'), owner='root', group='root', mode=0644, content=Template("tools-log4j.properties.j2")) generate_logfeeder_input_config( 'kafka', Template("input.config-kafka.json.j2", extra_imports=[default])) setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir) setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)
def setup_ranger_admin(upgrade_type=None): import params if upgrade_type is None: upgrade_type = Script.get_upgrade_type( default("/commandParams/upgrade_type", "")) ranger_home = params.ranger_home ranger_conf = params.ranger_conf Directory(ranger_conf, owner=params.unix_user, group=params.unix_group, create_parents=True) copy_jdbc_connector(ranger_home) File( format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content=DownloadSource( format("{jdk_location}/{check_db_connection_jar_name}")), mode=0644, ) generate_logfeeder_input_config( 'ranger', Template("input.config-ranger.json.j2", extra_imports=[default])) cp = format("{check_db_connection_jar}") if params.db_flavor.lower() == 'sqla': cp = cp + os.pathsep + format("{ranger_home}/ews/lib/sajdbc4.jar") else: cp = cp + os.pathsep + format("{driver_curl_target}") cp = cp + os.pathsep + format("{ranger_home}/ews/lib/*") db_connection_check_command = format( "{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_jdbc_connection_url}' {ranger_db_user} {ranger_db_password!p} {ranger_jdbc_driver}" ) env_dict = {} if params.db_flavor.lower() == 'sqla': env_dict = {'LD_LIBRARY_PATH': params.ld_lib_path} Execute(db_connection_check_command, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10, environment=env_dict) Execute( ('ln', '-sf', format('{ranger_home}/ews/webapp/WEB-INF/classes/conf'), format('{ranger_home}/conf')), not_if=format("ls {ranger_home}/conf"), only_if=format("ls {ranger_home}/ews/webapp/WEB-INF/classes/conf"), sudo=True) if upgrade_type is not None: src_file = format( '{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/ranger-admin-default-site.xml' ) dst_file = format('{ranger_home}/conf/ranger-admin-default-site.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) src_file = format( '{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/security-applicationContext.xml' ) dst_file = format('{ranger_home}/conf/security-applicationContext.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) Directory( format('{ranger_home}/'), owner=params.unix_user, group=params.unix_group, recursive_ownership=True, ) Directory(params.ranger_pid_dir, mode=0755, owner=params.unix_user, group=params.user_group, cd_access="a", create_parents=True) Directory(params.admin_log_dir, owner=params.unix_user, group=params.unix_group, create_parents=True, cd_access='a', mode=0755) if os.path.isfile(params.ranger_admin_default_file): File(params.ranger_admin_default_file, owner=params.unix_user, group=params.unix_group) else: Logger.warning( 'Required file {0} does not exist, copying the file to {1} path'. format(params.ranger_admin_default_file, ranger_conf)) src_file = format( '{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/ranger-admin-default-site.xml' ) dst_file = format('{ranger_home}/conf/ranger-admin-default-site.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) File(params.ranger_admin_default_file, owner=params.unix_user, group=params.unix_group) if os.path.isfile(params.security_app_context_file): File(params.security_app_context_file, owner=params.unix_user, group=params.unix_group) else: Logger.warning( 'Required file {0} does not exist, copying the file to {1} path'. format(params.security_app_context_file, ranger_conf)) src_file = format( '{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/security-applicationContext.xml' ) dst_file = format('{ranger_home}/conf/security-applicationContext.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) File(params.security_app_context_file, owner=params.unix_user, group=params.unix_group) if default( "/configurations/ranger-admin-site/ranger.authentication.method", "") == 'PAM': d = '/etc/pam.d' if os.path.isdir(d): if os.path.isfile(os.path.join(d, 'ranger-admin')): Logger.info('ranger-admin PAM file already exists.') else: File(format('{d}/ranger-admin'), content=Template('ranger_admin_pam.j2'), owner=params.unix_user, group=params.unix_group, mode=0644) if os.path.isfile(os.path.join(d, 'ranger-remote')): Logger.info('ranger-remote PAM file already exists.') else: File(format('{d}/ranger-remote'), content=Template('ranger_remote_pam.j2'), owner=params.unix_user, group=params.unix_group, mode=0644) else: Logger.error( "Unable to use PAM authentication, /etc/pam.d/ directory does not exist." ) # remove plain-text password from xml configs ranger_admin_site_copy = {} ranger_admin_site_copy.update( params.config['configurations']['ranger-admin-site']) for prop in params.ranger_admin_password_properties: if prop in ranger_admin_site_copy: ranger_admin_site_copy[prop] = "_" if 'ranger.ha.spnego.kerberos.keytab' in ranger_admin_site_copy: ranger_admin_site_copy[ 'ranger.spnego.kerberos.keytab'] = ranger_admin_site_copy[ 'ranger.ha.spnego.kerberos.keytab'] XmlConfig("ranger-admin-site.xml", conf_dir=ranger_conf, configurations=ranger_admin_site_copy, configuration_attributes=params.config['configurationAttributes'] ['ranger-admin-site'], owner=params.unix_user, group=params.unix_group, mode=0644) Directory( os.path.join(ranger_conf, 'ranger_jaas'), mode=0700, owner=params.unix_user, group=params.unix_group, ) if params.stack_supports_ranger_log4j: File(format('{ranger_home}/ews/webapp/WEB-INF/log4j.properties'), owner=params.unix_user, group=params.unix_group, content=InlineTemplate(params.admin_log4j), mode=0644) do_keystore_setup(upgrade_type=upgrade_type) create_core_site_xml(ranger_conf) if params.stack_supports_ranger_kerberos: if params.is_hbase_ha_enabled and params.ranger_hbase_plugin_enabled: XmlConfig( "hbase-site.xml", conf_dir=ranger_conf, configurations=params.config['configurations']['hbase-site'], configuration_attributes=params. config['configurationAttributes']['hbase-site'], owner=params.unix_user, group=params.unix_group, mode=0644) if params.is_namenode_ha_enabled and params.ranger_hdfs_plugin_enabled: XmlConfig( "hdfs-site.xml", conf_dir=ranger_conf, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params. config['configurationAttributes']['hdfs-site'], owner=params.unix_user, group=params.unix_group, mode=0644) File(format("{ranger_conf}/ranger-admin-env.sh"), content=InlineTemplate(params.ranger_env_content), owner=params.unix_user, group=params.unix_group, mode=0755)
def setup_hiveserver2(): import params File(params.start_hiveserver2_path, mode=0755, content=Template(format('{start_hiveserver2_script}'))) File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hiveserver2.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hiveserver2.properties.j2"), mode=0600) XmlConfig( "hiveserver2-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations']['hiveserver2-site'], configuration_attributes=params.config['configurationAttributes'] ['hiveserver2-site'], owner=params.hive_user, group=params.user_group, mode=0600) # ****** Begin Copy Tarballs ****** # ********************************* # if copy tarball to HDFS feature supported copy mapreduce.tar.gz and tez.tar.gz to HDFS if params.stack_version_formatted_major and check_stack_feature( StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major): copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs) copy_to_hdfs("tez", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs) # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode. # This can use a different source and dest location to account copy_to_hdfs("pig", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.pig_tar_source, custom_dest_file=params.pig_tar_dest_file, skip=params.sysprep_skip_copy_tarballs_hdfs) copy_to_hdfs("hive", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.hive_tar_source, custom_dest_file=params.hive_tar_dest_file, skip=params.sysprep_skip_copy_tarballs_hdfs) wildcard_tarballs = ["sqoop", "hadoop_streaming"] for tarball_name in wildcard_tarballs: source_file_pattern = eval("params." + tarball_name + "_tar_source") dest_dir = eval("params." + tarball_name + "_tar_dest_dir") if source_file_pattern is None or dest_dir is None: continue source_files = glob.glob( source_file_pattern) if "*" in source_file_pattern else [ source_file_pattern ] for source_file in source_files: src_filename = os.path.basename(source_file) dest_file = os.path.join(dest_dir, src_filename) copy_to_hdfs(tarball_name, params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=source_file, custom_dest_file=dest_file, skip=params.sysprep_skip_copy_tarballs_hdfs) # ******* End Copy Tarballs ******* # ********************************* # if warehouse directory is in DFS if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse( params.default_fs).scheme: if not is_empty(params.tez_hook_proto_base_directory): params.HdfsResource(params.tez_hook_proto_base_directory, type="directory", action="create_on_execute", owner=params.hive_user, mode=01755) if not is_empty(params.hive_hook_proto_base_directory): params.HdfsResource(params.hive_hook_proto_base_directory, type="directory", action="create_on_execute", owner=params.hive_user, mode=01777) dag_meta = params.tez_hook_proto_base_directory + "dag_meta" params.HdfsResource(dag_meta, type="directory", action="create_on_execute", owner=params.hive_user, mode=01777) dag_data = params.tez_hook_proto_base_directory + "dag_data" params.HdfsResource(dag_data, type="directory", action="create_on_execute", owner=params.hive_user, mode=01777) app_data = params.tez_hook_proto_base_directory + "app_data" params.HdfsResource(app_data, type="directory", action="create_on_execute", owner=params.hive_user, mode=01777) if not is_empty(params.hive_exec_scratchdir) and not urlparse( params.hive_exec_scratchdir).path.startswith("/tmp"): params.HdfsResource( params.hive_exec_scratchdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.hdfs_user, mode=0777 ) # Hive expects this dir to be writeable by everyone as it is used as a temp dir if params.hive_repl_cmrootdir is not None and params.hive_repl_cmrootdir.strip( ) != "": params.HdfsResource(params.hive_repl_cmrootdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=01777) if params.hive_repl_rootdir is not None and params.hive_repl_rootdir.strip( ) != "": params.HdfsResource(params.hive_repl_rootdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=0700) params.HdfsResource(None, action="execute") generate_logfeeder_input_config( 'hive', Template("input.config-hive.json.j2", extra_imports=[default]))
def hbase( name=None # 'master' or 'regionserver' or 'client' , action=None): import params Directory( params.hbase_conf_dir, owner=params.hbase_user, group=params.user_group, create_parents=True, recursive_ownership=True, ) Directory( params.hbase_tmp_dir, owner=params.hbase_user, cd_access="a", create_parents=True, recursive_ownership=True, ) Directory( os.path.join(params.local_dir, "jars"), owner=params.hbase_user, group=params.user_group, cd_access="a", mode=0775, create_parents=True) if params.hbase_wal_dir: Directory( params.hbase_wal_dir, owner=params.hbase_user, group=params.user_group, cd_access="a", create_parents=True, recursive_ownership=True, ) merged_ams_hbase_site = {} merged_ams_hbase_site.update( params.config['configurations']['ams-hbase-site']) if params.security_enabled: merged_ams_hbase_site.update( params.config['configurations']['ams-hbase-security-site']) if not params.is_hbase_distributed: File( format("{hbase_conf_dir}/core-site.xml"), action='delete', owner=params.hbase_user) File( format("{hbase_conf_dir}/hdfs-site.xml"), action='delete', owner=params.hbase_user) XmlConfig( "hbase-site.xml", conf_dir=params.hbase_conf_dir, configurations=merged_ams_hbase_site, configuration_attributes=params.config['configurationAttributes'] ['ams-hbase-site'], owner=params.hbase_user, group=params.user_group) # Phoenix spool file dir if not /tmp if not os.path.exists(params.phoenix_server_spool_dir): Directory( params.phoenix_server_spool_dir, owner=params.ams_user, mode=0755, group=params.user_group, cd_access="a", create_parents=True) pass if 'ams-hbase-policy' in params.config['configurations']: XmlConfig( "hbase-policy.xml", conf_dir=params.hbase_conf_dir, configurations=params.config['configurations']['ams-hbase-policy'], configuration_attributes=params.config['configurationAttributes'] ['ams-hbase-policy'], owner=params.hbase_user, group=params.user_group) # Manually overriding ownership of file installed by hadoop package else: File( format("{params.hbase_conf_dir}/hbase-policy.xml"), owner=params.hbase_user, group=params.user_group) File( format("{hbase_conf_dir}/hbase-env.sh"), owner=params.hbase_user, content=InlineTemplate(params.hbase_env_sh_template)) # Metrics properties File( os.path.join(params.hbase_conf_dir, "hadoop-metrics2-hbase.properties"), owner=params.hbase_user, group=params.user_group, content=Template("hadoop-metrics2-hbase.properties.j2")) # hbase_TemplateConfig( params.metric_prop_file_name, # tag = 'GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS' # ) hbase_TemplateConfig('regionservers', user=params.hbase_user) if params.security_enabled: hbase_TemplateConfig( format("hbase_{name}_jaas.conf"), user=params.hbase_user) hbase_TemplateConfig( format("hbase_client_jaas.conf"), user=params.hbase_user) hbase_TemplateConfig( format("ams_zookeeper_jaas.conf"), user=params.hbase_user) if name != "client": Directory( params.hbase_pid_dir, owner=params.hbase_user, create_parents=True, cd_access="a", mode=0755, ) Directory( params.hbase_log_dir, owner=params.hbase_user, create_parents=True, cd_access="a", mode=0755, ) if name == "master": if not params.is_local_fs_rootdir: # If executing Stop All, HDFS is probably down if action != 'stop' and not params.skip_create_hbase_root_dir: params.HdfsResource( params.hbase_root_dir, type="directory", action="create_on_execute", owner=params.hbase_user, mode=0775, dfs_type=params.dfs_type) params.HdfsResource( params.hbase_staging_dir, type="directory", action="create_on_execute", owner=params.hbase_user, mode=0711, dfs_type=params.dfs_type) params.HdfsResource(None, action="execute") if params.is_hbase_distributed: # Workaround for status commands not aware of operating mode File( format("{params.hbase_pid_dir}/distributed_mode"), action="create", mode=0644, owner=params.hbase_user) pass else: local_root_dir = params.hbase_root_dir # cut protocol name if local_root_dir.startswith("file://"): local_root_dir = local_root_dir[7:] # otherwise assume dir name is provided as is Directory( local_root_dir, owner=params.hbase_user, cd_access="a", create_parents=True, recursive_ownership=True) File( format("{params.hbase_pid_dir}/distributed_mode"), action="delete", owner=params.hbase_user) if params.hbase_log4j_props is not None: File( format("{params.hbase_conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.hbase_user, content=InlineTemplate(params.hbase_log4j_props)) elif os.path.exists(format("{params.hbase_conf_dir}/log4j.properties")): File( format("{params.hbase_conf_dir}/log4j.properties"), mode=0644, group=params.user_group, owner=params.hbase_user) generate_logfeeder_input_config( 'ambari-metrics', Template( "input.config-ambari-metrics.json.j2", extra_imports=[default]))