def is_https_enabled_in_hdfs(dfs_http_policy, dfs_https_enable): https_enabled = False if not is_empty(dfs_http_policy): https_enabled = dfs_http_policy.lower() == "https_only" elif not is_empty(dfs_https_enable): https_enabled = dfs_https_enable return https_enabled
def is_ha_enabled(hdfs_site): dfs_ha_nameservices = get_nameservice(hdfs_site) if not dfs_ha_nameservices or is_empty(dfs_ha_nameservices): return False dfs_ha_namenode_ids = hdfs_site[format("dfs.ha.namenodes.{dfs_ha_nameservices}")] if not is_empty(dfs_ha_namenode_ids): dfs_ha_namemodes_ids_list = dfs_ha_namenode_ids.split(",") dfs_ha_namenode_ids_array_len = len(dfs_ha_namemodes_ids_list) if dfs_ha_namenode_ids_array_len > 1: return True return False
def do_keystore_setup(rolling_upgrade=False): import params ranger_home = params.ranger_home cred_lib_path = params.cred_lib_path cred_setup_prefix = params.cred_setup_prefix if rolling_upgrade: ranger_home = format("/usr/hdp/{version}/ranger-admin") cred_lib_path = os.path.join(ranger_home,"cred","lib","*") cred_setup_prefix = (format('{ranger_home}/ranger_credential_helper.py'), '-l', cred_lib_path) if not is_empty(params.ranger_credential_provider_path): jceks_path = params.ranger_credential_provider_path cred_setup = cred_setup_prefix + ('-f', jceks_path, '-k', params.ranger_jpa_jdbc_credential_alias, '-v', PasswordString(params.ranger_ambari_db_password), '-c', '1') Execute(cred_setup, environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True ) File(params.ranger_credential_provider_path, owner = params.unix_user, group = params.unix_group, mode = 0640 ) if not is_empty(params.ranger_credential_provider_path) and (params.ranger_audit_source_type).lower() == 'db' and not is_empty(params.ranger_ambari_audit_db_password): jceks_path = params.ranger_credential_provider_path cred_setup = cred_setup_prefix + ('-f', jceks_path, '-k', params.ranger_jpa_audit_jdbc_credential_alias, '-v', PasswordString(params.ranger_ambari_audit_db_password), '-c', '1') Execute(cred_setup, environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True ) File(params.ranger_credential_provider_path, owner = params.unix_user, group = params.unix_group, mode = 0640 )
def get_path_from_url(address): """ Return port from URL. If the address is numeric, the address is assumed to be a port and is returned. If address is UnknownConfiguration, UnknownConfiguration will be returned. """ if is_empty(address): return address result = re.findall("^((.+)://)?(([a-zA-Z0-9]|\.|-)*)(:([\d]{2,}))?/(.*)$", address) if result: return result[0][6] return None
def get_namenode_states_noretries(hdfs_site, security_enabled, run_user): """ return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....] """ active_namenodes = [] standby_namenodes = [] unknown_namenodes = [] name_service = hdfs_site['dfs.nameservices'] nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service # now we have something like 'nn1,nn2,nn3,nn4' # turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id] # ie dfs.namenode.http-address.hacluster.nn1 nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',') for nn_unique_id in nn_unique_ids: is_https_enabled = hdfs_site['dfs.https.enable'] if not is_empty(hdfs_site['dfs.https.enable']) else False if not is_https_enabled: key = NAMENODE_HTTP_FRAGMENT.format(name_service,nn_unique_id) protocol = "http" else: key = NAMENODE_HTTPS_FRAGMENT.format(name_service,nn_unique_id) protocol = "https" if key in hdfs_site: # use str() to ensure that unicode strings do not have the u' in them value = str(hdfs_site[key]) jmx_uri = JMX_URI_FRAGMENT.format(protocol, value) state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled, run_user, is_https_enabled) # If JMX parsing failed if not state: check_service_cmd = "hdfs haadmin -getServiceState {0}".format(nn_unique_id) code, out = shell.call(check_service_cmd, logoutput=True, user=run_user) if code == 0 and out: if HDFS_NN_STATE_STANDBY in out: state = HDFS_NN_STATE_STANDBY elif HDFS_NN_STATE_ACTIVE in out: state = HDFS_NN_STATE_ACTIVE if state == HDFS_NN_STATE_ACTIVE: active_namenodes.append((nn_unique_id, value)) elif state == HDFS_NN_STATE_STANDBY: standby_namenodes.append((nn_unique_id, value)) else: unknown_namenodes.append((nn_unique_id, value)) return active_namenodes, standby_namenodes, unknown_namenodes
def get_port_from_url(address): """ Return port from URL. If the address is numeric, the address is assumed to be a port and is returned. If address is UnknownConfiguration, UnknownConfiguration will be returned. If no port was found, Fail will be raised. """ if is_empty(address): return address if isinstance(address, (int, long)): return address port = re.findall(":([\d]{1,5})(?=/|$)", address) if port: return port[0] elif address.isdigit(): return address raise Fail("No port in URL:{0}".format(address))
'xml_configurations_supported'] ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] ranger_admin_log_dir = default( "/configurations/ranger-env/ranger_admin_log_dir", "/var/log/ranger/admin") #need to set the defaut to false to satisfy downgrade from 4.2,5 to 4.2 or 4.1 is_supported_solr_ranger = default( '/configurations/solr-env/is_supported_solr_ranger', False) #ranger solr properties if has_ranger_admin and is_supported_solr_ranger: enable_ranger_solr = config['configurations'][ 'ranger-solr-plugin-properties']['ranger-solr-plugin-enabled'] enable_ranger_solr = not is_empty( enable_ranger_solr) and enable_ranger_solr.lower() == 'yes' policymgr_mgr_url = config['configurations']['admin-properties'][ 'policymgr_external_url'] if 'admin-properties' in config[ 'configurations'] and 'policymgr_external_url' in config[ 'configurations'][ 'admin-properties'] and policymgr_mgr_url.endswith('/'): policymgr_mgr_url = policymgr_mgr_url.rstrip('/') xa_audit_db_flavor = config['configurations']['admin-properties'][ 'DB_FLAVOR'] xa_audit_db_flavor = xa_audit_db_flavor.lower( ) if xa_audit_db_flavor else None xa_audit_db_name = default( '/configurations/admin-properties/audit_db_name', 'ranger_audits') xa_audit_db_user = default( '/configurations/admin-properties/audit_db_user', 'rangerlogger')
repo_config_username = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] if security_enabled: sn_principal_name = default("/configurations/hdfs-site/dfs.secondary.namenode.kerberos.principal", "nn/[email protected]") sn_principal_name = sn_principal_name.replace('_HOST',hostname.lower()) ranger_env = config['configurations']['ranger-env'] ranger_plugin_properties = config['configurations']['ranger-hdfs-plugin-properties'] policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_user'] #For curl command in ranger plugin to get db connector jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' is_https_enabled = config['configurations']['hdfs-site']['dfs.https.enable'] if \ not is_empty(config['configurations']['hdfs-site']['dfs.https.enable']) else False if has_ranger_admin: enable_ranger_hdfs = (config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled'].lower() == 'yes') xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) repo_config_password = unicode(config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']) xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() if xa_audit_db_flavor == 'mysql': jdbc_symlink_name = "mysql-jdbc-driver.jar" jdbc_jar_name = "mysql-connector-java.jar" audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}') jdbc_driver = "com.mysql.jdbc.Driver" elif xa_audit_db_flavor == 'oracle': jdbc_jar_name = "ojdbc6.jar" jdbc_symlink_name = "oracle-jdbc-driver.jar"
usgsync_log4j_file = format('{ranger_ugsync_conf}/log4j.xml') if stack_supports_ranger_log4j: usgsync_log4j_file = format('{ranger_ugsync_conf}/log4j.properties') cred_validator_file = format('{usersync_home}/native/credValidator.uexe') ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() usersync_exturl = config['configurations']['admin-properties'][ 'policymgr_external_url'] if usersync_exturl.endswith('/'): usersync_exturl = usersync_exturl.rstrip('/') ranger_host = config['clusterHostInfo']['ranger_admin_hosts'][0] ugsync_host = 'localhost' usersync_host_info = config['clusterHostInfo']['ranger_usersync_hosts'] if not is_empty(usersync_host_info) and len(usersync_host_info) > 0: ugsync_host = config['clusterHostInfo']['ranger_usersync_hosts'][0] ranger_external_url = config['configurations']['admin-properties'][ 'policymgr_external_url'] if ranger_external_url.endswith('/'): ranger_external_url = ranger_external_url.rstrip('/') ranger_db_name = config['configurations']['admin-properties']['db_name'] ranger_auditdb_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits') sql_command_invoker = config['configurations']['admin-properties'][ 'SQL_COMMAND_INVOKER'] db_root_user = config['configurations']['admin-properties']['db_root_user'] db_root_password = unicode( config['configurations']['admin-properties']['db_root_password']) db_host = config['configurations']['admin-properties']['db_host']
def do_keystore_setup(upgrade_type=None): import params ranger_home = params.ranger_home cred_lib_path = params.cred_lib_path if not is_empty(params.ranger_credential_provider_path): ranger_credential_helper(cred_lib_path, params.ranger_jpa_jdbc_credential_alias, params.ranger_ambari_db_password, params.ranger_credential_provider_path) File(params.ranger_credential_provider_path, owner = params.unix_user, group = params.unix_group, mode = 0640 ) if not is_empty(params.ranger_credential_provider_path) and (params.ranger_audit_source_type).lower() == 'db' and not is_empty(params.ranger_ambari_audit_db_password): ranger_credential_helper(cred_lib_path, params.ranger_jpa_audit_jdbc_credential_alias, params.ranger_ambari_audit_db_password, params.ranger_credential_provider_path) File(params.ranger_credential_provider_path, owner = params.unix_user, group = params.unix_group, mode = 0640 ) if params.ranger_auth_method.upper() == "LDAP": ranger_ldap_auth_password = params.ranger_usersync_ldap_ldapbindpassword if params.ranger_ldap_bind_auth_password != "{{ranger_usersync_ldap_ldapbindpassword}}": ranger_ldap_auth_password = params.ranger_ldap_bind_auth_password ranger_credential_helper(params.cred_lib_path, params.ranger_ldap_password_alias, ranger_ldap_auth_password, params.ranger_credential_provider_path) File(params.ranger_credential_provider_path, owner = params.unix_user, group = params.unix_group, mode = 0640 ) if params.ranger_auth_method.upper() == "ACTIVE_DIRECTORY": ranger_ad_auth_password = params.ranger_usersync_ldap_ldapbindpassword if params.ranger_ad_bind_auth_password != "{{ranger_usersync_ldap_ldapbindpassword}}": ranger_ad_auth_password = params.ranger_ad_bind_auth_password ranger_credential_helper(params.cred_lib_path, params.ranger_ad_password_alias, ranger_ad_auth_password, params.ranger_credential_provider_path) File(params.ranger_credential_provider_path, owner = params.unix_user, group = params.unix_group, mode = 0640 ) if params.stack_supports_secure_ssl_password: ranger_credential_helper(params.cred_lib_path, params.ranger_truststore_alias, params.truststore_password, params.ranger_credential_provider_path) if params.https_enabled and not params.http_enabled: ranger_credential_helper(params.cred_lib_path, params.ranger_https_keystore_alias, params.https_keystore_password, params.ranger_credential_provider_path) File(params.ranger_credential_provider_path, owner = params.unix_user, group = params.unix_group, mode = 0640 )
def setup_usersync(upgrade_type=None): import params usersync_home = params.usersync_home ranger_home = params.ranger_home ranger_ugsync_conf = params.ranger_ugsync_conf if not is_empty(params.ranger_usersync_ldap_ldapbindpassword) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder': password_validation(params.ranger_usersync_ldap_ldapbindpassword) Directory(params.ranger_pid_dir, mode=0750, owner = params.unix_user, group = params.unix_group ) Directory(params.usersync_log_dir, owner = params.unix_user, group = params.unix_group, cd_access = 'a', create_parents=True, mode=0755 ) File(format('{ranger_ugsync_conf}/ranger-usersync-env-logdir.sh'), content = format("export logdir={usersync_log_dir}"), owner = params.unix_user, group = params.unix_group, mode=0755 ) Directory(format("{ranger_ugsync_conf}/"), owner = params.unix_user ) if upgrade_type is not None: src_file = format('{usersync_home}/conf.dist/ranger-ugsync-default.xml') dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) if params.stack_supports_ranger_log4j: File(format('{usersync_home}/conf/log4j.properties'), owner=params.unix_user, group=params.unix_group, content=params.usersync_log4j, mode=0644 ) elif upgrade_type is not None and not params.stack_supports_ranger_log4j: src_file = format('{usersync_home}/conf.dist/log4j.xml') dst_file = format('{usersync_home}/conf/log4j.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) XmlConfig("ranger-ugsync-site.xml", conf_dir=ranger_ugsync_conf, configurations=params.config['configurations']['ranger-ugsync-site'], configuration_attributes=params.config['configuration_attributes']['ranger-ugsync-site'], owner=params.unix_user, group=params.unix_group, mode=0644) if os.path.isfile(params.ranger_ugsync_default_file): File(params.ranger_ugsync_default_file, owner=params.unix_user, group=params.unix_group) if os.path.isfile(params.usgsync_log4j_file): File(params.usgsync_log4j_file, owner=params.unix_user, group=params.unix_group) if os.path.isfile(params.cred_validator_file): File(params.cred_validator_file, group=params.unix_group, mode=04555) cred_lib = os.path.join(usersync_home,"lib","*") ranger_credential_helper(cred_lib, 'usersync.ssl.key.password', params.ranger_usersync_keystore_password, params.ugsync_jceks_path) if not is_empty(params.ranger_usersync_ldap_ldapbindpassword) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder': ranger_credential_helper(cred_lib, 'ranger.usersync.ldap.bindalias', params.ranger_usersync_ldap_ldapbindpassword, params.ugsync_jceks_path) ranger_credential_helper(cred_lib, 'usersync.ssl.truststore.password', params.ranger_usersync_truststore_password, params.ugsync_jceks_path) File(params.ugsync_jceks_path, owner = params.unix_user, group = params.unix_group, mode = 0640 ) File([params.usersync_start, params.usersync_stop], owner = params.unix_user, group = params.unix_group ) File(params.usersync_services_file, mode = 0755, ) Execute(('ln','-sf', format('{usersync_services_file}'),'/usr/bin/ranger-usersync'), not_if=format("ls /usr/bin/ranger-usersync"), only_if=format("ls {usersync_services_file}"), sudo=True) if not os.path.isfile(params.ranger_usersync_keystore_file): cmd = format("{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'") Execute(cmd, logoutput=True, user = params.unix_user) File(params.ranger_usersync_keystore_file, owner = params.unix_user, group = params.unix_group, mode = 0640 ) create_core_site_xml(ranger_ugsync_conf)
mysql_host = config['clusterHostInfo']['metron_enrichment_mysql_server_hosts'][ 0] mysql_port = config['configurations']['metron-env'][ 'metron_enrichment_db_port'] mysql_adduser_path = tmp_dir + "/addMysqlUser.sh" mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh" mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh" enrichment_metron_user = config['configurations']['metron-env'][ 'metron_enrichment_db_user'] enrichment_metron_user_passwd = config['configurations']['metron-env'][ 'metron_enrichment_db_password'] enrichment_metron_user_passwd = unicode( enrichment_metron_user_passwd) if not is_empty( enrichment_metron_user_passwd) else enrichment_metron_user_passwd mysql_process_name = status_params.mysql_process_name # create partial functions with common arguments for every HdfsResource call # to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code HdfsResource = functools.partial( HdfsResource, user=hdfs_user, hdfs_resource_ignore_file= "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled=security_enabled, keytab=hdfs_user_keytab, kinit_path_local=kinit_path_local, hadoop_bin_dir=hadoop_bin_dir, hadoop_conf_dir=hadoop_conf_dir, principal_name=hdfs_principal_name,
enable_ranger_kafka = default( "configurations/ranger-kafka-plugin-properties/ranger-kafka-plugin-enabled", "No") enable_ranger_kafka = True if enable_ranger_kafka.lower() == 'yes' else False # ranger kafka-plugin supported flag, instead of dependending on is_supported_kafka_ranger/kafka-env.xml, using stack feature is_supported_kafka_ranger = check_stack_feature( StackFeature.KAFKA_RANGER_PLUGIN_SUPPORT, version_for_stack_feature_checks) # ranger kafka properties if enable_ranger_kafka and is_supported_kafka_ranger: # get ranger policy url policymgr_mgr_url = config['configurations']['ranger-kafka-security'][ 'ranger.plugin.kafka.policy.rest.url'] if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'): policymgr_mgr_url = policymgr_mgr_url.rstrip('/') # ranger audit db user xa_audit_db_user = default( '/configurations/admin-properties/audit_db_user', 'rangerlogger') xa_audit_db_password = '' if not is_empty( config['configurations']['admin-properties']['audit_db_password'] ) and stack_supports_ranger_audit_db and has_ranger_admin: xa_audit_db_password = config['configurations']['admin-properties'][ 'audit_db_password'] # ranger kafka service/repository name repo_name = str(config['clusterName']) + '_kafka'
ranger_env = config['configurations']['ranger-env'] ranger_plugin_properties = config['configurations'][ 'ranger-hbase-plugin-properties'] policy_user = config['configurations']['ranger-hbase-plugin-properties'][ 'policy_user'] #For curl command in ranger plugin to get db connector jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' enable_ranger_hbase = False if has_ranger_admin: enable_ranger_hbase = ( config['configurations']['ranger-hbase-plugin-properties'] ['ranger-hbase-plugin-enabled'].lower() == 'yes') xa_audit_db_password = '' if not is_empty(config['configurations']['admin-properties'] ['audit_db_password']) and stack_supports_ranger_audit_db: xa_audit_db_password = unicode( config['configurations']['admin-properties']['audit_db_password']) repo_config_password = unicode( config['configurations']['ranger-hbase-plugin-properties'] ['REPOSITORY_CONFIG_PASSWORD']) xa_audit_db_flavor = ( config['configurations']['admin-properties']['DB_FLAVOR']).lower() previous_jdbc_jar_name = None if xa_audit_db_flavor == 'mysql': jdbc_symlink_name = "mysql-jdbc-driver.jar" jdbc_jar_name = "mysql-connector-java.jar" previous_jdbc_jar_name = default( "/hostLevelParams/previous_custom_mysql_jdbc_name", None) audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
has_ranger_admin = not len(ranger_admin_hosts) == 0 xml_configurations_supported = config['configurations']['ranger-env'][ 'xml_configurations_supported'] ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] ranger_admin_log_dir = default( "/configurations/ranger-env/ranger_admin_log_dir", "/var/log/ranger/admin") is_supported_kafka_ranger = config['configurations']['kafka-env'][ 'is_supported_kafka_ranger'] #ranger kafka properties if has_ranger_admin and is_supported_kafka_ranger: enable_ranger_kafka = config['configurations'][ 'ranger-kafka-plugin-properties']['ranger-kafka-plugin-enabled'] enable_ranger_kafka = not is_empty( enable_ranger_kafka) and enable_ranger_kafka.lower() == 'yes' policymgr_mgr_url = config['configurations']['admin-properties'][ 'policymgr_external_url'] sql_connector_jar = config['configurations']['admin-properties'][ 'SQL_CONNECTOR_JAR'] xa_audit_db_flavor = config['configurations']['admin-properties'][ 'DB_FLAVOR'] xa_audit_db_flavor = xa_audit_db_flavor.lower( ) if xa_audit_db_flavor else None xa_audit_db_name = config['configurations']['admin-properties'][ 'audit_db_name'] xa_audit_db_user = config['configurations']['admin-properties'][ 'audit_db_user'] xa_audit_db_password = unicode( config['configurations']['admin-properties']['audit_db_password']) xa_db_host = config['configurations']['admin-properties']['db_host']
usersync_log_dir = default("/configurations/ranger-env/ranger_usersync_log_dir", "/var/log/ranger/usersync") admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir", "/var/log/ranger/admin") ranger_admin_default_file = format('{ranger_conf}/ranger-admin-default-site.xml') security_app_context_file = format('{ranger_conf}/security-applicationContext.xml') ranger_ugsync_default_file = format('{ranger_ugsync_conf}/ranger-ugsync-default.xml') usgsync_log4j_file = format('{ranger_ugsync_conf}/log4j.xml') cred_validator_file = format('{usersync_home}/native/credValidator.uexe') ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() usersync_exturl = config['configurations']['admin-properties']['policymgr_external_url'] ranger_host = config['clusterHostInfo']['ranger_admin_hosts'][0] ugsync_host = 'localhost' usersync_host_info = config['clusterHostInfo']['ranger_usersync_hosts'] if not is_empty(usersync_host_info) and len(usersync_host_info) > 0: ugsync_host = config['clusterHostInfo']['ranger_usersync_hosts'][0] ranger_external_url = config['configurations']['admin-properties']['policymgr_external_url'] ranger_db_name = config['configurations']['admin-properties']['db_name'] ranger_auditdb_name = config['configurations']['admin-properties']['audit_db_name'] sql_command_invoker = config['configurations']['admin-properties']['SQL_COMMAND_INVOKER'] db_root_user = config['configurations']['admin-properties']['db_root_user'] db_root_password = unicode(config['configurations']['admin-properties']['db_root_password']) db_host = config['configurations']['admin-properties']['db_host'] ranger_db_user = config['configurations']['admin-properties']['db_user'] ranger_audit_db_user = config['configurations']['admin-properties']['audit_db_user'] ranger_db_password = unicode(config['configurations']['admin-properties']['db_password']) #ranger-env properties oracle_home = default("/configurations/ranger-env/oracle_home", "-")
def get_namenode_states_noretries(hdfs_site, security_enabled, run_user): """ return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....] """ active_namenodes = [] standby_namenodes = [] unknown_namenodes = [] name_service = hdfs_site['dfs.nameservices'] nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service # now we have something like 'nn1,nn2,nn3,nn4' # turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id] # ie dfs.namenode.http-address.hacluster.nn1 nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',') for nn_unique_id in nn_unique_ids: is_https_enabled = hdfs_site['dfs.https.enable'] if not is_empty( hdfs_site['dfs.https.enable']) else False rpc_key = NAMENODE_RPC_FRAGMENT.format(name_service, nn_unique_id) if not is_https_enabled: key = NAMENODE_HTTP_FRAGMENT.format(name_service, nn_unique_id) protocol = "http" else: key = NAMENODE_HTTPS_FRAGMENT.format(name_service, nn_unique_id) protocol = "https" if key in hdfs_site: # use str() to ensure that unicode strings do not have the u' in them value = str(hdfs_site[key]) if INADDR_ANY in value and rpc_key in hdfs_site: rpc_value = str(hdfs_site[rpc_key]) if INADDR_ANY not in rpc_value: rpc_host = rpc_value.split(":")[0] value = value.replace(INADDR_ANY, rpc_host) jmx_uri = JMX_URI_FRAGMENT.format(protocol, value) state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled, run_user, is_https_enabled) # If JMX parsing failed if not state: check_service_cmd = "hdfs haadmin -getServiceState {0}".format( nn_unique_id) code, out = shell.call(check_service_cmd, logoutput=True, user=run_user) if code == 0 and out: if HDFS_NN_STATE_STANDBY in out: state = HDFS_NN_STATE_STANDBY elif HDFS_NN_STATE_ACTIVE in out: state = HDFS_NN_STATE_ACTIVE if state == HDFS_NN_STATE_ACTIVE: active_namenodes.append((nn_unique_id, value)) elif state == HDFS_NN_STATE_STANDBY: standby_namenodes.append((nn_unique_id, value)) else: unknown_namenodes.append((nn_unique_id, value)) return active_namenodes, standby_namenodes, unknown_namenodes
def hive(name=None): import params if name == 'hiveserver2': # copy tarball to HDFS feature not supported if not (params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major)): params.HdfsResource(params.webhcat_apps_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=0755) # Create webhcat dirs. if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: params.HdfsResource(params.hcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=params.hcat_hdfs_user_mode) params.HdfsResource(params.webhcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=params.webhcat_hdfs_user_mode) # ****** Begin Copy Tarballs ****** # ********************************* # if copy tarball to HDFS feature supported copy mapreduce.tar.gz and tez.tar.gz to HDFS if params.stack_version_formatted_major and check_stack_feature( StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major): copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode. # This can use a different source and dest location to account copy_to_hdfs("pig", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.pig_tar_source, custom_dest_file=params.pig_tar_dest_file, host_sys_prepped=params.host_sys_prepped) copy_to_hdfs("hive", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.hive_tar_source, custom_dest_file=params.hive_tar_dest_file, host_sys_prepped=params.host_sys_prepped) wildcard_tarballs = ["sqoop", "hadoop_streaming"] for tarball_name in wildcard_tarballs: source_file_pattern = eval("params." + tarball_name + "_tar_source") dest_dir = eval("params." + tarball_name + "_tar_dest_dir") if source_file_pattern is None or dest_dir is None: continue source_files = glob.glob( source_file_pattern) if "*" in source_file_pattern else [ source_file_pattern ] for source_file in source_files: src_filename = os.path.basename(source_file) dest_file = os.path.join(dest_dir, src_filename) copy_to_hdfs(tarball_name, params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=source_file, custom_dest_file=dest_file, host_sys_prepped=params.host_sys_prepped) # ******* End Copy Tarballs ******* # ********************************* # if warehouse directory is in DFS if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse( params.default_fs).scheme: # Create Hive Metastore Warehouse Dir params.HdfsResource(params.hive_apps_whs_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=0777) else: Logger.info( format( "Not creating warehouse directory '{hive_apps_whs_dir}', as the location is not in DFS." )) # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=params.hive_hdfs_user_mode) if not is_empty(params.hive_exec_scratchdir) and not urlparse( params.hive_exec_scratchdir).path.startswith("/tmp"): params.HdfsResource( params.hive_exec_scratchdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.hdfs_user, mode=0777 ) # Hive expects this dir to be writeable by everyone as it is used as a temp dir params.HdfsResource(None, action="execute") Directory(params.hive_etc_dir_prefix, mode=0755) # We should change configurations for client as well as for server. # The reason is that stale-configs are service-level, not component. Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list)) for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) XmlConfig( "hive-site.xml", conf_dir=params.hive_config_dir, configurations=params.hive_site_config, configuration_attributes=params.config['configuration_attributes'] ['hive-site'], owner=params.hive_user, group=params.user_group, mode=0644) # Generate atlas-application.properties.xml file if has_atlas_in_cluster(): atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) if name == 'hiveserver2': XmlConfig( "hiveserver2-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations']['hiveserver2-site'], configuration_attributes=params.config['configuration_attributes'] ['hiveserver2-site'], owner=params.hive_user, group=params.user_group, mode=0644) if params.hive_metastore_site_supported and name == 'metastore': XmlConfig( "hivemetastore-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations'] ['hivemetastore-site'], configuration_attributes=params.config['configuration_attributes'] ['hivemetastore-site'], owner=params.hive_user, group=params.user_group, mode=0644) File(format("{hive_config_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, content=InlineTemplate(params.hive_env_sh_template)) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2")) if name == 'metastore' or name == 'hiveserver2': if params.hive_jdbc_target is not None and not os.path.exists( params.hive_jdbc_target): jdbc_connector(params.hive_jdbc_target, params.hive_previous_jdbc_jar) if params.hive2_jdbc_target is not None and not os.path.exists( params.hive2_jdbc_target): jdbc_connector(params.hive2_jdbc_target, params.hive2_previous_jdbc_jar) File( format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content=DownloadSource( format("{jdk_location}{check_db_connection_jar_name}")), mode=0644, ) if name == 'metastore': File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hivemetastore.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hivemetastore.properties.j2")) File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh')) if params.init_metastore_schema: create_schema_cmd = format( "export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -initSchema " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} -verbose") check_schema_created_cmd = as_user( format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_schematool_bin}/schematool -info " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user) # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. # Fixing it with the hack below: quoted_hive_metastore_user_passwd = quote_bash_args( quote_bash_args(params.hive_metastore_user_passwd)) if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[ 1:-1] Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr( check_schema_created_cmd.replace( format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) Execute(create_schema_cmd, not_if=check_schema_created_cmd, user=params.hive_user) elif name == 'hiveserver2': File(params.start_hiveserver2_path, mode=0755, content=Template(format('{start_hiveserver2_script}'))) File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hiveserver2.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hiveserver2.properties.j2")) if name != "client": Directory(params.hive_pid_dir, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) Directory(params.hive_log_dir, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) Directory(params.hive_var_lib, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755)
def enable_kms_plugin(): import params if params.has_ranger_admin: ranger_flag = False if params.stack_supports_ranger_kerberos and params.security_enabled: if not is_empty(params.rangerkms_principal ) and params.rangerkms_principal != '': ranger_flag = check_ranger_service_support_kerberos( params.kms_user, params.rangerkms_keytab, params.rangerkms_principal) else: ranger_flag = check_ranger_service_support_kerberos( params.kms_user, params.spengo_keytab, params.spnego_principal) else: ranger_flag = check_ranger_service() if not ranger_flag: Logger.error('Error in Get/Create service for Ranger Kms.') current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S") File(format('{kms_conf_dir}/ranger-security.xml'), owner=params.kms_user, group=params.kms_group, mode=0644, content=format( '<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>')) Directory([ os.path.join('/etc', 'ranger', params.repo_name), os.path.join('/etc', 'ranger', params.repo_name, 'policycache') ], owner=params.kms_user, group=params.kms_group, mode=0775, create_parents=True) File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache', format('kms_{repo_name}.json')), owner=params.kms_user, group=params.kms_group, mode=0644) # remove plain-text password from xml configs plugin_audit_properties_copy = {} plugin_audit_properties_copy.update( params.config['configurations']['ranger-kms-audit']) if params.plugin_audit_password_property in plugin_audit_properties_copy: plugin_audit_properties_copy[ params.plugin_audit_password_property] = "crypted" XmlConfig( "ranger-kms-audit.xml", conf_dir=params.kms_conf_dir, configurations=plugin_audit_properties_copy, configuration_attributes=params.config['configuration_attributes'] ['ranger-kms-audit'], owner=params.kms_user, group=params.kms_group, mode=0744) XmlConfig( "ranger-kms-security.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations'] ['ranger-kms-security'], configuration_attributes=params.config['configuration_attributes'] ['ranger-kms-security'], owner=params.kms_user, group=params.kms_group, mode=0744) # remove plain-text password from xml configs ranger_kms_policymgr_ssl_copy = {} ranger_kms_policymgr_ssl_copy.update( params.config['configurations']['ranger-kms-policymgr-ssl']) for prop in params.kms_plugin_password_properties: if prop in ranger_kms_policymgr_ssl_copy: ranger_kms_policymgr_ssl_copy[prop] = "crypted" XmlConfig( "ranger-policymgr-ssl.xml", conf_dir=params.kms_conf_dir, configurations=ranger_kms_policymgr_ssl_copy, configuration_attributes=params.config['configuration_attributes'] ['ranger-kms-policymgr-ssl'], owner=params.kms_user, group=params.kms_group, mode=0744) if params.xa_audit_db_is_enabled: cred_setup = params.cred_setup_prefix + ( '-f', params.credential_file, '-k', 'auditDBCred', '-v', PasswordString(params.xa_audit_db_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = params.cred_setup_prefix + ( '-f', params.credential_file, '-k', 'sslKeyStore', '-v', PasswordString(params.ssl_keystore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = params.cred_setup_prefix + ( '-f', params.credential_file, '-k', 'sslTrustStore', '-v', PasswordString(params.ssl_truststore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) File(params.credential_file, owner=params.kms_user, group=params.kms_group, mode=0640) # create ranger kms audit directory if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node: params.HdfsResource("/ranger/audit", type="directory", action="create_on_execute", owner=params.hdfs_user, group=params.hdfs_user, mode=0755, recursive_chmod=True) params.HdfsResource("/ranger/audit/kms", type="directory", action="create_on_execute", owner=params.kms_user, group=params.kms_group, mode=0750, recursive_chmod=True) params.HdfsResource(None, action="execute") if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1: Logger.info( 'Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml' ) XmlConfig( "hdfs-site.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params. config['configuration_attributes']['hdfs-site'], owner=params.kms_user, group=params.kms_group, mode=0644) else: File(format('{kms_conf_dir}/hdfs-site.xml'), action="delete")
usersync_log_dir = default("/configurations/ranger-env/ranger_usersync_log_dir", "/var/log/ranger/usersync") admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir", "/var/log/ranger/admin") ranger_admin_default_file = format('{ranger_conf}/ranger-admin-default-site.xml') security_app_context_file = format('{ranger_conf}/security-applicationContext.xml') ranger_ugsync_default_file = format('{ranger_ugsync_conf}/ranger-ugsync-default.xml') usgsync_log4j_file = format('{ranger_ugsync_conf}/log4j.xml') cred_validator_file = format('{usersync_home}/native/credValidator.uexe') ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() usersync_exturl = config['configurations']['admin-properties']['policymgr_external_url'] ranger_host = config['clusterHostInfo']['ranger_admin_hosts'][0] ugsync_host = 'localhost' usersync_host_info = config['clusterHostInfo']['ranger_usersync_hosts'] if not is_empty(usersync_host_info) and len(usersync_host_info) > 0: ugsync_host = config['clusterHostInfo']['ranger_usersync_hosts'][0] ranger_external_url = config['configurations']['admin-properties']['policymgr_external_url'] ranger_db_name = config['configurations']['admin-properties']['db_name'] ranger_auditdb_name = config['configurations']['admin-properties']['audit_db_name'] sql_command_invoker = config['configurations']['admin-properties']['SQL_COMMAND_INVOKER'] db_root_user = config['configurations']['admin-properties']['db_root_user'] db_root_password = unicode(config['configurations']['admin-properties']['db_root_password']) db_host = config['configurations']['admin-properties']['db_host'] ranger_db_user = config['configurations']['admin-properties']['db_user'] ranger_audit_db_user = config['configurations']['admin-properties']['audit_db_user'] ranger_db_password = unicode(config['configurations']['admin-properties']['db_password']) #ranger-env properties oracle_home = default("/configurations/ranger-env/oracle_home", "-")
repo_config_username = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] if security_enabled: sn_principal_name = default("/configurations/hdfs-site/dfs.secondary.namenode.kerberos.principal", "nn/[email protected]") sn_principal_name = sn_principal_name.replace('_HOST',hostname.lower()) ranger_env = config['configurations']['ranger-env'] ranger_plugin_properties = config['configurations']['ranger-hdfs-plugin-properties'] policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_user'] #For curl command in ranger plugin to get db connector jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' is_https_enabled = config['configurations']['hdfs-site']['dfs.https.enable'] if \ not is_empty(config['configurations']['hdfs-site']['dfs.https.enable']) else False if has_ranger_admin: enable_ranger_hdfs = (config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled'].lower() == 'yes') xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) repo_config_password = unicode(config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']) xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() if xa_audit_db_flavor == 'mysql': jdbc_symlink_name = "mysql-jdbc-driver.jar" jdbc_jar_name = "mysql-connector-java.jar" audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}') jdbc_driver = "com.mysql.jdbc.Driver" elif xa_audit_db_flavor == 'oracle': jdbc_jar_name = "ojdbc6.jar" jdbc_symlink_name = "oracle-jdbc-driver.jar"
def hive_interactive(name=None): import params MB_TO_BYTES = 1048576 # if warehouse directory is in DFS if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse( params.default_fs).scheme: # Create Hive Metastore Warehouse Dir params.HdfsResource(params.hive_metastore_warehouse_dir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=0700) # create directories for llap package pkg_dir = '/user/' + params.hive_user + '/.yarn' for dir in [pkg_dir, pkg_dir + '/package', pkg_dir + '/package/LLAP']: # hdfsresouces handles parent creation badly params.HdfsResource(dir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.user_group, mode=0755) if not is_empty(params.tez_hook_proto_base_directory): params.HdfsResource(params.tez_hook_proto_base_directory, type="directory", action="create_on_execute", owner=params.hive_user, mode=01755) if not is_empty(params.hive_hook_proto_base_directory): params.HdfsResource(params.hive_hook_proto_base_directory, type="directory", action="create_on_execute", owner=params.hive_user, mode=01777) dag_meta = params.tez_hook_proto_base_directory + "dag_meta" params.HdfsResource(dag_meta, type="directory", action="create_on_execute", owner=params.hive_user, mode=01777) dag_data = params.tez_hook_proto_base_directory + "dag_data" params.HdfsResource(dag_data, type="directory", action="create_on_execute", owner=params.hive_user, mode=01777) app_data = params.tez_hook_proto_base_directory + "app_data" params.HdfsResource(app_data, type="directory", action="create_on_execute", owner=params.hive_user, mode=01777) else: Logger.info( format( "Not creating warehouse directory '{hive_metastore_warehouse_dir}', as the location is not in DFS." )) # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=params.hive_hdfs_user_mode) # list of properties that should be excluded from the config # this approach is a compromise against adding a dedicated config # type for hive_server_interactive or needed config groups on a # per component basis exclude_list = ['hive.enforce.bucketing', 'hive.enforce.sorting'] # List of configs to be excluded from hive2 client, but present in Hive2 server. exclude_list_for_hive2_client = [ 'javax.jdo.option.ConnectionPassword', 'hadoop.security.credential.provider.path' ] Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list)) for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) ''' As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml, we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml. ''' merged_hive_interactive_site = {} merged_hive_interactive_site.update(params.hive_site_config) merged_hive_interactive_site.update( params.config['configurations']['hive-interactive-site']) for item in exclude_list: if item in merged_hive_interactive_site.keys(): del merged_hive_interactive_site[item] merged_hive_interactive_site[ 'hive.llap.daemon.vcpus.per.instance'] = format( merged_hive_interactive_site['hive.llap.daemon.vcpus.per.instance'] ) merged_hive_interactive_site[ 'hive.server2.active.passive.ha.enable'] = str( params.hive_server_interactive_ha).lower() ''' Config 'hive.llap.io.memory.size' calculated value in stack_advisor is in MB as of now. We need to convert it to bytes before we write it down to config file. ''' if 'hive.llap.io.memory.size' in merged_hive_interactive_site.keys(): hive_llap_io_mem_size_in_mb = merged_hive_interactive_site.get( "hive.llap.io.memory.size") hive_llap_io_mem_size_in_bytes = long( hive_llap_io_mem_size_in_mb) * MB_TO_BYTES merged_hive_interactive_site[ 'hive.llap.io.memory.size'] = hive_llap_io_mem_size_in_bytes Logger.info( "Converted 'hive.llap.io.memory.size' value from '{0} MB' to '{1} Bytes' before writing " "it to config file.".format(hive_llap_io_mem_size_in_mb, hive_llap_io_mem_size_in_bytes)) ''' Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook', which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed. ''' # Generate atlas-application.properties.xml file if params.enable_atlas_hook and params.stack_supports_atlas_hook_for_hive_interactive: Logger.info("Setup for Atlas Hive2 Hook started.") atlas_hook_filepath = os.path.join( params.hive_server_interactive_conf_dir, params.atlas_hook_filename) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group) Logger.info("Setup for Atlas Hive2 Hook done.") else: # Required for HDP 2.5 stacks Logger.info( "Skipping setup for Atlas Hook, as it is disabled/ not supported.") remove_atlas_hook_if_exists(merged_hive_interactive_site) ''' As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml, we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml. ''' merged_tez_interactive_site = {} if 'tez-site' in params.config['configurations']: merged_tez_interactive_site.update( params.config['configurations']['tez-site']) Logger.info( "Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'." ) else: Logger.error( "Tez's 'tez-site' couldn't be retrieved from passed-in configurations." ) merged_tez_interactive_site.update( params.config['configurations']['tez-interactive-site']) XmlConfig("tez-site.xml", conf_dir=params.tez_interactive_conf_dir, configurations=merged_tez_interactive_site, configuration_attributes=params.config['configurationAttributes'] ['tez-interactive-site'], owner=params.tez_interactive_user, group=params.user_group, mode=0664) ''' Merge properties from hiveserver2-interactive-site into hiveserver2-site ''' merged_hiveserver2_interactive_site = {} if 'hiveserver2-site' in params.config['configurations']: merged_hiveserver2_interactive_site.update( params.config['configurations']['hiveserver2-site']) Logger.info( "Retrieved 'hiveserver2-site' for merging with 'hiveserver2-interactive-site'." ) else: Logger.error( "'hiveserver2-site' couldn't be retrieved from passed-in configurations." ) merged_hiveserver2_interactive_site.update( params.config['configurations']['hiveserver2-interactive-site']) # Create config files under hive_server_interactive_conf_dir: # hive-site.xml # hive-env.sh # llap-daemon-log4j2.properties # llap-cli-log4j2.properties # hive-log4j2.properties # hive-exec-log4j2.properties # beeline-log4j2.properties hive_server_interactive_conf_dir = params.hive_server_interactive_conf_dir mode_identified = 0600 merged_hive_interactive_site = update_credential_provider_path( merged_hive_interactive_site, 'hive-site', os.path.join(conf_dir, 'hive-site.jceks'), params.hive_user, params.user_group) XmlConfig("hive-site.xml", conf_dir=hive_server_interactive_conf_dir, configurations=merged_hive_interactive_site, configuration_attributes=params.config['configurationAttributes'] ['hive-interactive-site'], owner=params.hive_user, group=params.user_group, mode=0644) XmlConfig("hiveserver2-site.xml", conf_dir=hive_server_interactive_conf_dir, configurations=merged_hiveserver2_interactive_site, configuration_attributes=params.config['configurationAttributes'] ['hiveserver2-interactive-site'], owner=params.hive_user, group=params.user_group, mode=mode_identified) File(format("{hive_server_interactive_conf_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, mode=0755, content=InlineTemplate(params.hive_interactive_env_sh_template)) llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties' File(format( "{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.llap_daemon_log4j)) llap_cli_log4j2_filename = 'llap-cli-log4j2.properties' File(format( "{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.llap_cli_log4j2)) hive_log4j2_filename = 'hive-log4j2.properties' File(format("{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.hive_log4j2)) hive_exec_log4j2_filename = 'hive-exec-log4j2.properties' File(format( "{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.hive_exec_log4j2)) beeline_log4j2_filename = 'beeline-log4j2.properties' File( format("{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"), mode=mode_identified, group=params.user_group, owner=params.hive_user, content=InlineTemplate(params.beeline_log4j2)) XmlConfig("beeline-site.xml", conf_dir=conf_dir, configurations=params.beeline_site_config, owner=params.hive_user, group=params.user_group, mode=mode_identified) File(os.path.join(hive_server_interactive_conf_dir, "hadoop-metrics2-hiveserver2.properties"), owner=params.hive_user, group=params.user_group, mode=mode_identified, content=Template("hadoop-metrics2-hiveserver2.properties.j2")) File(format( "{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties" ), owner=params.hive_user, group=params.user_group, mode=mode_identified, content=Template("hadoop-metrics2-llapdaemon.j2")) File(format( "{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties" ), owner=params.hive_user, group=params.user_group, mode=mode_identified, content=Template("hadoop-metrics2-llaptaskscheduler.j2")) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2")) if not os.path.exists(params.target_hive_interactive): jdbc_connector(params.target_hive_interactive, params.hive_intaractive_previous_jdbc_jar) File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content=DownloadSource( format("{jdk_location}/{check_db_connection_jar_name}")), mode=0644) File(params.start_hiveserver2_interactive_path, mode=0755, content=Template(format('{start_hiveserver2_interactive_script}'))) Directory(params.hive_pid_dir, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) Directory(params.hive_log_dir, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) Directory(params.hive_interactive_var_lib, create_parents=True, cd_access='a', owner=params.hive_user, group=params.user_group, mode=0755) generate_logfeeder_input_config( 'hive', Template("input.config-hive.json.j2", extra_imports=[default]))
def yarn(name=None, config_dir=None): """ :param name: Component name, apptimelineserver, nodemanager, resourcemanager, or None (defaults for client) :param config_dir: Which config directory to write configs to, which could be different during rolling upgrade. """ import params if config_dir is None: config_dir = params.hadoop_conf_dir if name == "historyserver": if params.yarn_log_aggregation_enabled: params.HdfsResource(params.yarn_nm_app_log_dir, action="create_on_execute", type="directory", owner=params.yarn_user, group=params.user_group, mode=01777, recursive_chmod=True) # create the /tmp folder with proper permissions if it doesn't exist yet if params.entity_file_history_directory.startswith('/tmp'): params.HdfsResource( params.hdfs_tmp_dir, action="create_on_execute", type="directory", owner=params.hdfs_user, mode=0777, ) params.HdfsResource(params.entity_file_history_directory, action="create_on_execute", type="directory", owner=params.yarn_user, group=params.user_group) params.HdfsResource("/mapred", type="directory", action="create_on_execute", owner=params.mapred_user) params.HdfsResource("/mapred/system", type="directory", action="create_on_execute", owner=params.hdfs_user) params.HdfsResource(params.mapreduce_jobhistory_done_dir, type="directory", action="create_on_execute", owner=params.mapred_user, group=params.user_group, change_permissions_for_parents=True, mode=0777) params.HdfsResource(None, action="execute") Directory( params.jhs_leveldb_state_store_dir, owner=params.mapred_user, group=params.user_group, create_parents=True, cd_access="a", recursive_ownership=True, ) #<editor-fold desc="Node Manager Section"> if name == "nodemanager": # First start after enabling/disabling security if params.toggle_nm_security: Directory(params.nm_local_dirs_list + params.nm_log_dirs_list, action='delete') # If yarn.nodemanager.recovery.dir exists, remove this dir if params.yarn_nodemanager_recovery_dir: Directory(InlineTemplate( params.yarn_nodemanager_recovery_dir).get_content(), action='delete') # Setting NM marker file if params.security_enabled: Directory(params.nm_security_marker_dir) File( params.nm_security_marker, content= "Marker file to track first start after enabling/disabling security. " "During first start yarn local, log dirs are removed and recreated" ) elif not params.security_enabled: File(params.nm_security_marker, action="delete") if not params.security_enabled or params.toggle_nm_security: # handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount. nm_log_dir_to_mount_file_content = handle_mounted_dirs( create_log_dir, params.nm_log_dirs, params.nm_log_dir_to_mount_file, params) # create a history file used by handle_mounted_dirs File(params.nm_log_dir_to_mount_file, owner=params.hdfs_user, group=params.user_group, mode=0644, content=nm_log_dir_to_mount_file_content) nm_local_dir_to_mount_file_content = handle_mounted_dirs( create_local_dir, params.nm_local_dirs, params.nm_local_dir_to_mount_file, params) File(params.nm_local_dir_to_mount_file, owner=params.hdfs_user, group=params.user_group, mode=0644, content=nm_local_dir_to_mount_file_content) #</editor-fold> if params.yarn_nodemanager_recovery_dir: Directory( InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(), owner=params.yarn_user, group=params.user_group, create_parents=True, mode=0755, cd_access='a', ) Directory( [params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir], owner=params.yarn_user, group=params.user_group, create_parents=True, cd_access='a', ) Directory( [ params.mapred_pid_dir_prefix, params.mapred_pid_dir, params.mapred_log_dir_prefix, params.mapred_log_dir ], owner=params.mapred_user, group=params.user_group, create_parents=True, cd_access='a', ) Directory( [params.yarn_log_dir_prefix], owner=params.yarn_user, group=params.user_group, create_parents=True, ignore_failures=True, cd_access='a', ) XmlConfig( "core-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['core-site'], configuration_attributes=params.config['configuration_attributes'] ['core-site'], owner=params.hdfs_user, group=params.user_group, mode=0644) # During RU, Core Masters and Slaves need hdfs-site.xml # TODO, instead of specifying individual configs, which is susceptible to breaking when new configs are added, # RU should rely on all available in <stack-root>/<version>/hadoop/conf if 'hdfs-site' in params.config['configurations']: XmlConfig( "hdfs-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params.config['configuration_attributes'] ['hdfs-site'], owner=params.hdfs_user, group=params.user_group, mode=0644) XmlConfig( "mapred-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['mapred-site'], configuration_attributes=params.config['configuration_attributes'] ['mapred-site'], owner=params.yarn_user, group=params.user_group, mode=0644) XmlConfig( "yarn-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['yarn-site'], configuration_attributes=params.config['configuration_attributes'] ['yarn-site'], owner=params.yarn_user, group=params.user_group, mode=0644) XmlConfig( "capacity-scheduler.xml", conf_dir=config_dir, configurations=params.config['configurations']['capacity-scheduler'], configuration_attributes=params.config['configuration_attributes'] ['capacity-scheduler'], owner=params.yarn_user, group=params.user_group, mode=0644) if name == 'resourcemanager': Directory( params.rm_nodes_exclude_dir, mode=0755, create_parents=True, cd_access='a', ) File(params.rm_nodes_exclude_path, owner=params.yarn_user, group=params.user_group) File(params.yarn_job_summary_log, owner=params.yarn_user, group=params.user_group) if not is_empty( params.node_label_enable ) and params.node_label_enable or is_empty( params.node_label_enable) and params.node_labels_dir: params.HdfsResource(params.node_labels_dir, type="directory", action="create_on_execute", change_permissions_for_parents=True, owner=params.yarn_user, group=params.user_group, mode=0700) params.HdfsResource(None, action="execute") elif name == 'apptimelineserver': Directory( params.ats_leveldb_dir, owner=params.yarn_user, group=params.user_group, create_parents=True, cd_access="a", ) # if stack support application timeline-service state store property (timeline_state_store stack feature) if params.stack_supports_timeline_state_store: Directory( params.ats_leveldb_state_store_dir, owner=params.yarn_user, group=params.user_group, create_parents=True, cd_access="a", ) # app timeline server 1.5 directories if not is_empty(params.entity_groupfs_store_dir): parent_path = os.path.dirname(params.entity_groupfs_store_dir) params.HdfsResource(parent_path, type="directory", action="create_on_execute", change_permissions_for_parents=True, owner=params.yarn_user, group=params.user_group, mode=0755) params.HdfsResource(params.entity_groupfs_store_dir, type="directory", action="create_on_execute", owner=params.yarn_user, group=params.user_group, mode=params.entity_groupfs_store_dir_mode) if not is_empty(params.entity_groupfs_active_dir): parent_path = os.path.dirname(params.entity_groupfs_active_dir) params.HdfsResource(parent_path, type="directory", action="create_on_execute", change_permissions_for_parents=True, owner=params.yarn_user, group=params.user_group, mode=0755) params.HdfsResource(params.entity_groupfs_active_dir, type="directory", action="create_on_execute", owner=params.yarn_user, group=params.user_group, mode=params.entity_groupfs_active_dir_mode) params.HdfsResource(None, action="execute") File(format("{limits_conf_dir}/yarn.conf"), mode=0644, content=Template('yarn.conf.j2')) File(format("{limits_conf_dir}/mapreduce.conf"), mode=0644, content=Template('mapreduce.conf.j2')) File(os.path.join(config_dir, "yarn-env.sh"), owner=params.yarn_user, group=params.user_group, mode=0755, content=InlineTemplate(params.yarn_env_sh_template)) container_executor = format("{yarn_container_bin}/container-executor") File(container_executor, group=params.yarn_executor_container_group, mode=params.container_executor_mode) File(os.path.join(config_dir, "container-executor.cfg"), group=params.user_group, mode=0644, content=Template('container-executor.cfg.j2')) Directory(params.cgroups_dir, group=params.user_group, create_parents=True, mode=0755, cd_access="a") if params.security_enabled: tc_mode = 0644 tc_owner = "root" else: tc_mode = None tc_owner = params.hdfs_user File(os.path.join(config_dir, "mapred-env.sh"), owner=tc_owner, mode=0755, content=InlineTemplate(params.mapred_env_sh_template)) if params.security_enabled: File(os.path.join(params.hadoop_bin, "task-controller"), owner="root", group=params.mapred_tt_group, mode=06050) File(os.path.join(config_dir, 'taskcontroller.cfg'), owner=tc_owner, mode=tc_mode, group=params.mapred_tt_group, content=Template("taskcontroller.cfg.j2")) else: File(os.path.join(config_dir, 'taskcontroller.cfg'), owner=tc_owner, content=Template("taskcontroller.cfg.j2")) if "mapred-site" in params.config['configurations']: XmlConfig( "mapred-site.xml", conf_dir=config_dir, configurations=params.config['configurations']['mapred-site'], configuration_attributes=params.config['configuration_attributes'] ['mapred-site'], owner=params.mapred_user, group=params.user_group) if "capacity-scheduler" in params.config['configurations']: XmlConfig( "capacity-scheduler.xml", conf_dir=config_dir, configurations=params.config['configurations'] ['capacity-scheduler'], configuration_attributes=params.config['configuration_attributes'] ['capacity-scheduler'], owner=params.hdfs_user, group=params.user_group) if "ssl-client" in params.config['configurations']: XmlConfig( "ssl-client.xml", conf_dir=config_dir, configurations=params.config['configurations']['ssl-client'], configuration_attributes=params.config['configuration_attributes'] ['ssl-client'], owner=params.hdfs_user, group=params.user_group) Directory( params.hadoop_conf_secure_dir, create_parents=True, owner='root', group=params.user_group, cd_access='a', ) XmlConfig( "ssl-client.xml", conf_dir=params.hadoop_conf_secure_dir, configurations=params.config['configurations']['ssl-client'], configuration_attributes=params.config['configuration_attributes'] ['ssl-client'], owner=params.hdfs_user, group=params.user_group) if "ssl-server" in params.config['configurations']: XmlConfig( "ssl-server.xml", conf_dir=config_dir, configurations=params.config['configurations']['ssl-server'], configuration_attributes=params.config['configuration_attributes'] ['ssl-server'], owner=params.hdfs_user, group=params.user_group) if os.path.exists(os.path.join(config_dir, 'fair-scheduler.xml')): File(os.path.join(config_dir, 'fair-scheduler.xml'), owner=params.mapred_user, group=params.user_group) if os.path.exists(os.path.join(config_dir, 'ssl-client.xml.example')): File(os.path.join(config_dir, 'ssl-client.xml.example'), owner=params.mapred_user, group=params.user_group) if os.path.exists(os.path.join(config_dir, 'ssl-server.xml.example')): File(os.path.join(config_dir, 'ssl-server.xml.example'), owner=params.mapred_user, group=params.user_group)
def setup_ranger_hive_metastore_service(): """ Creates ranger hive service in ranger admin installed in same cluster for cluster depolyed in cloud env. """ import params if params.has_ranger_admin and params.ranger_hive_metastore_lookup: repo_name = str(params.config['clusterName']) + '_hive' repo_name_value = params.config['configurations'][ 'ranger-hive-security']['ranger.plugin.hive.service.name'] if not is_empty( repo_name_value) and repo_name_value != "{{repo_name}}": repo_name = repo_name_value hive_ranger_plugin_config = { 'username': params.config['configurations']['ranger-hive-plugin-properties'] ['REPOSITORY_CONFIG_USERNAME'], 'password': params.config['configurations']['ranger-hive-plugin-properties'] ['REPOSITORY_CONFIG_PASSWORD'], 'jdbc.driverClassName': params.config['configurations']['ranger-hive-plugin-properties'] ['jdbc.driverClassName'], 'jdbc.url': 'none', 'commonNameForCertificate': params.config['configurations']['ranger-hive-plugin-properties'] ['common.name.for.certificate'], 'ambari.service.check.user': params.config['configurations']['ranger-hive-plugin-properties'] ['policy_user'] } if params.security_enabled: hive_ranger_plugin_config[ 'policy.download.auth.users'] = params.hive_user hive_ranger_plugin_config[ 'tag.download.auth.users'] = params.hive_user hive_ranger_plugin_config[ 'policy.grantrevoke.auth.users'] = params.hive_user custom_ranger_service_config = generate_ranger_service_config( params.config['configurations']['ranger-hive-plugin-properties']) if len(custom_ranger_service_config) > 0: hive_ranger_plugin_config.update(custom_ranger_service_config) hive_ranger_plugin_repo = { 'isEnabled': 'true', 'configs': hive_ranger_plugin_config, 'description': 'Hive service', 'name': repo_name, 'type': 'hive' } ranger_admin_obj = RangeradminV2( url=params.config['configurations']['ranger-hive-security'] ['ranger.plugin.hive.policy.rest.url'], skip_if_rangeradmin_down=not params.retryAble) ranger_admin_obj.create_ranger_repository( component='hive', repo_name=repo_name, repo_properties=hive_ranger_plugin_repo, ambari_ranger_admin=params.config['configurations']['ranger-env'] ['ranger_admin_username'], ambari_ranger_password=params.config['configurations'] ['ranger-env']['ranger_admin_password'], admin_uname=params.config['configurations']['ranger-env'] ['admin_username'], admin_password=params.config['configurations']['ranger-env'] ['admin_password'], policy_user=params.config['configurations'] ['ranger-hive-plugin-properties']['policy_user'], is_security_enabled=params.security_enabled, is_stack_supports_ranger_kerberos=params. stack_supports_ranger_kerberos, component_user=params.hive_user, component_user_principal=params.hive_metastore_principal_with_host if params.security_enabled else None, component_user_keytab=params.hive_metastore_keytab_path if params.security_enabled else None)
def hive(name=None): import params if name == 'hiveserver2': # HDP 2.1.* or lower if params.hdp_stack_version_major != "" and compare_versions( params.hdp_stack_version_major, "2.2.0.0") < 0: params.HdfsResource(params.webhcat_apps_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=0755) # Create webhcat dirs. if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: params.HdfsResource(params.hcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hcat_user, mode=params.hcat_hdfs_user_mode) params.HdfsResource(params.webhcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=params.webhcat_hdfs_user_mode) # ****** Begin Copy Tarballs ****** # ********************************* # HDP 2.2 or higher, copy mapreduce.tar.gz to HDFS if params.hdp_stack_version_major != "" and compare_versions( params.hdp_stack_version_major, '2.2') >= 0: copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user) copy_to_hdfs("tez", params.user_group, params.hdfs_user) # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode. # This can use a different source and dest location to account for both HDP 2.1 and 2.2 copy_to_hdfs("pig", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.pig_tar_source, custom_dest_file=params.pig_tar_dest_file) copy_to_hdfs("hive", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.hive_tar_source, custom_dest_file=params.hive_tar_dest_file) wildcard_tarballs = ["sqoop", "hadoop_streaming"] for tarball_name in wildcard_tarballs: source_file_pattern = eval("params." + tarball_name + "_tar_source") dest_dir = eval("params." + tarball_name + "_tar_dest_dir") if source_file_pattern is None or dest_dir is None: continue source_files = glob.glob( source_file_pattern) if "*" in source_file_pattern else [ source_file_pattern ] for source_file in source_files: src_filename = os.path.basename(source_file) dest_file = os.path.join(dest_dir, src_filename) copy_to_hdfs(tarball_name, params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=source_file, custom_dest_file=dest_file) # ******* End Copy Tarballs ******* # ********************************* # Create Hive Metastore Warehouse Dir params.HdfsResource(params.hive_apps_whs_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=0777) # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=params.hive_hdfs_user_mode) if not is_empty(params.hive_exec_scratchdir) and not urlparse( params.hive_exec_scratchdir).path.startswith("/tmp"): params.HdfsResource( params.hive_exec_scratchdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.hdfs_user, mode=0777 ) # Hive expects this dir to be writeable by everyone as it is used as a temp dir params.HdfsResource(None, action="execute") Directory(params.hive_etc_dir_prefix, mode=0755) # We should change configurations for client as well as for server. # The reason is that stale-configs are service-level, not component. for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) XmlConfig( "hive-site.xml", conf_dir=params.hive_config_dir, configurations=params.hive_site_config, configuration_attributes=params.config['configuration_attributes'] ['hive-site'], owner=params.hive_user, group=params.user_group, mode=0644) setup_atlas_hive() if params.hive_specific_configs_supported and name == 'hiveserver2': XmlConfig( "hiveserver2-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations']['hiveserver2-site'], configuration_attributes=params.config['configuration_attributes'] ['hiveserver2-site'], owner=params.hive_user, group=params.user_group, mode=0644) File(format("{hive_config_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, content=InlineTemplate(params.hive_env_sh_template)) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, recursive=True, owner='root', group='root') File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2")) if name == 'metastore' or name == 'hiveserver2': jdbc_connector() File( format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content=DownloadSource( format("{jdk_location}{check_db_connection_jar_name}")), mode=0644, ) if name == 'metastore': File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh')) if params.init_metastore_schema: create_schema_cmd = format( "export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_bin}/schematool -initSchema " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p}") check_schema_created_cmd = as_user( format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_bin}/schematool -info " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p}"), params.hive_user) Execute(create_schema_cmd, not_if=check_schema_created_cmd, user=params.hive_user) elif name == 'hiveserver2': File(params.start_hiveserver2_path, mode=0755, content=Template(format('{start_hiveserver2_script}'))) if name != "client": crt_directory(params.hive_pid_dir) crt_directory(params.hive_log_dir) crt_directory(params.hive_var_lib)
# upgrades would cause these directories to have a version instead of "current" # which would cause a lot of problems when writing out hadoop-env.sh; instead # force the use of "current" in the hook hdfs_user_nofile_limit = default( "/configurations/hadoop-env/hdfs_user_nofile_limit", "128000") hadoop_home = stack_select.get_hadoop_dir("home") hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec") hadoop_lib_home = stack_select.get_hadoop_dir("lib") hadoop_dir = "/etc/hadoop" hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir") datanode_max_locked_memory = config['configurations']['hdfs-site'][ 'dfs.datanode.max.locked.memory'] is_datanode_max_locked_memory_set = not is_empty( config['configurations']['hdfs-site']['dfs.datanode.max.locked.memory']) mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*" if not security_enabled: hadoop_secure_dn_user = '******' else: dfs_dn_port = get_port(dfs_dn_addr) dfs_dn_http_port = get_port(dfs_dn_http_addr) dfs_dn_https_port = get_port(dfs_dn_https_addr) # We try to avoid inability to start datanode as a plain user due to usage of root-owned ports if dfs_http_policy == "HTTPS_ONLY": secure_dn_ports_are_in_use = is_secure_port( dfs_dn_port) or is_secure_port(dfs_dn_https_port) elif dfs_http_policy == "HTTP_AND_HTTPS": secure_dn_ports_are_in_use = is_secure_port(
def setup_hiveserver2(): import params File(params.start_hiveserver2_path, mode=0755, content=Template(format('{start_hiveserver2_script}'))) File(os.path.join(params.hive_server_conf_dir, "hadoop-metrics2-hiveserver2.properties"), owner=params.hive_user, group=params.user_group, content=Template("hadoop-metrics2-hiveserver2.properties.j2"), mode=0600) XmlConfig( "hiveserver2-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations']['hiveserver2-site'], configuration_attributes=params.config['configuration_attributes'] ['hiveserver2-site'], owner=params.hive_user, group=params.user_group, mode=0600) # copy tarball to HDFS feature not supported params.HdfsResource(params.webhcat_apps_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=0755) # Create webhcat dirs. if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: params.HdfsResource(params.hcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=params.hcat_hdfs_user_mode) params.HdfsResource(params.webhcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=params.webhcat_hdfs_user_mode) # if warehouse directory is in DFS if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse( params.default_fs).scheme: # Create Hive Metastore Warehouse Dir params.HdfsResource(params.hive_apps_whs_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=0777) else: Logger.info( format( "Not creating warehouse directory '{hive_apps_whs_dir}', as the location is not in DFS." )) # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=params.hive_hdfs_user_mode) if not is_empty(params.hive_exec_scratchdir) and not urlparse( params.hive_exec_scratchdir).path.startswith("/tmp"): params.HdfsResource( params.hive_exec_scratchdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.hdfs_user, mode=0777 ) # Hive expects this dir to be writeable by everyone as it is used as a temp dir params.HdfsResource(None, action="execute")
def setup_tagsync(upgrade_type=None): import params ranger_tagsync_home = params.ranger_tagsync_home ranger_home = params.ranger_home ranger_tagsync_conf = params.ranger_tagsync_conf tagsync_log4j_file = format('{ranger_tagsync_conf}/log4j.xml') Directory(format("{ranger_tagsync_conf}"), owner = params.unix_user, group = params.unix_group, create_parents = True ) Directory(params.ranger_pid_dir, mode=0750, create_parents=True, owner = params.unix_user, group = params.unix_group, cd_access = "a", ) Directory(params.tagsync_log_dir, create_parents = True, owner = params.unix_user, group = params.unix_group, cd_access = "a", mode=0755 ) File(format('{ranger_tagsync_conf}/ranger-tagsync-env-logdir.sh'), content = format("export RANGER_TAGSYNC_LOG_DIR={tagsync_log_dir}"), owner = params.unix_user, group = params.unix_group, mode=0755 ) XmlConfig("ranger-tagsync-site.xml", conf_dir=ranger_tagsync_conf, configurations=params.config['configurations']['ranger-tagsync-site'], configuration_attributes=params.config['configuration_attributes']['ranger-tagsync-site'], owner=params.unix_user, group=params.unix_group, mode=0644) PropertiesFile(format('{ranger_tagsync_conf}/application.properties'), properties = params.tagsync_application_properties, mode=0755, owner=params.unix_user, group=params.unix_group ) if params.stack_supports_ranger_log4j: File(format('{ranger_tagsync_conf}/log4j.properties'), owner=params.unix_user, group=params.unix_group, content=params.tagsync_log4j, mode=0644 ) src_file = format('{ranger_tagsync_home}/conf.dist/log4j.xml') dst_file = format('{tagsync_log4j_file}') Execute(('cp', '-f', src_file, dst_file), sudo=True) if os.path.isfile(tagsync_log4j_file): File(tagsync_log4j_file, owner=params.unix_user, group=params.unix_group) else: Logger.warning('Required file {0} does not exist, copying the file to {1} path'.format(tagsync_log4j_file, ranger_tagsync_conf)) src_file = format('{ranger_tagsync_home}/conf.dist/log4j.xml') dst_file = format('{tagsync_log4j_file}') Execute(('cp', '-f', src_file, dst_file), sudo=True) File(tagsync_log4j_file, owner=params.unix_user, group=params.unix_group) cred_lib = os.path.join(ranger_tagsync_home,"lib","*") if not is_empty(params.tagsync_jceks_path) and not is_empty(params.ranger_tagsync_tagadmin_password) and params.tagsync_enabled: ranger_credential_helper(cred_lib, 'tagadmin.user.password', params.ranger_tagsync_tagadmin_password, params.tagsync_jceks_path) File(params.tagsync_jceks_path, owner = params.unix_user, group = params.unix_group, mode = 0640 ) File(params.tagsync_services_file, mode = 0755, ) Execute(('ln','-sf', format('{tagsync_services_file}'),'/usr/bin/ranger-tagsync'), not_if=format("ls /usr/bin/ranger-tagsync"), only_if=format("ls {tagsync_services_file}"), sudo=True) create_core_site_xml(ranger_tagsync_conf)
if check_stack_feature(StackFeature.HIVE_WEBHCAT_SPECIFIC_CONFIGS, version_for_stack_feature_checks): # this is NOT a typo. Configs for hcatalog/webhcat point to a # specific directory which is NOT called 'conf' hcat_conf_dir = format('{stack_root}/current/hive-webhcat/etc/hcatalog') config_dir = format('{stack_root}/current/hive-webhcat/etc/webhcat') if check_stack_feature(StackFeature.HIVE_METASTORE_SITE_SUPPORT, version_for_stack_feature_checks): hive_metastore_site_supported = True execute_path = os.environ['PATH'] + os.pathsep + hive_bin + os.pathsep + hadoop_bin_dir hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName'] hive_jdbc_connection_url = config['configurations']['hive-site']['javax.jdo.option.ConnectionURL'] hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword'] hive_metastore_user_passwd = unicode(hive_metastore_user_passwd) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd hive_metastore_db_type = config['configurations']['hive-env']['hive_database_type'] #HACK Temporarily use dbType=azuredb while invoking schematool if hive_metastore_db_type == "mssql": hive_metastore_db_type = "azuredb" #users hive_user = config['configurations']['hive-env']['hive_user'] #JDBC driver jar name hive_jdbc_driver = config['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName'] jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' hive_database_name = config['configurations']['hive-env']['hive_database_name'] hive_database = config['configurations']['hive-env']['hive_database']
def setup_usersync(upgrade_type=None): import params usersync_home = params.usersync_home ranger_home = params.ranger_home ranger_ugsync_conf = params.ranger_ugsync_conf if not is_empty(params.ranger_usersync_ldap_ldapbindpassword) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder': password_validation(params.ranger_usersync_ldap_ldapbindpassword) Directory(params.ranger_pid_dir, mode=0755, owner = params.unix_user, group = params.user_group, cd_access = "a", create_parents=True ) if params.stack_supports_pid: File(format('{ranger_ugsync_conf}/ranger-usersync-env-piddir.sh'), content = format("export USERSYNC_PID_DIR_PATH={ranger_pid_dir}\nexport UNIX_USERSYNC_USER={unix_user}"), owner = params.unix_user, group = params.unix_group, mode=0755 ) Directory(params.usersync_log_dir, owner = params.unix_user, group = params.unix_group, cd_access = 'a', create_parents=True, mode=0755, recursive_ownership = True ) File(format('{ranger_ugsync_conf}/ranger-usersync-env-logdir.sh'), content = format("export logdir={usersync_log_dir}"), owner = params.unix_user, group = params.unix_group, mode=0755 ) Directory(format("{ranger_ugsync_conf}/"), owner = params.unix_user ) if upgrade_type is not None: src_file = format('{usersync_home}/conf.dist/ranger-ugsync-default.xml') dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) if params.stack_supports_ranger_log4j: File(format('{usersync_home}/conf/log4j.properties'), owner=params.unix_user, group=params.unix_group, content=InlineTemplate(params.usersync_log4j), mode=0644 ) elif upgrade_type is not None and not params.stack_supports_ranger_log4j: src_file = format('{usersync_home}/conf.dist/log4j.xml') dst_file = format('{usersync_home}/conf/log4j.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) # remove plain-text password from xml configs ranger_ugsync_site_copy = {} ranger_ugsync_site_copy.update(params.config['configurations']['ranger-ugsync-site']) for prop in params.ranger_usersync_password_properties: if prop in ranger_ugsync_site_copy: ranger_ugsync_site_copy[prop] = "_" XmlConfig("ranger-ugsync-site.xml", conf_dir=ranger_ugsync_conf, configurations=ranger_ugsync_site_copy, configuration_attributes=params.config['configuration_attributes']['ranger-ugsync-site'], owner=params.unix_user, group=params.unix_group, mode=0644) if os.path.isfile(params.ranger_ugsync_default_file): File(params.ranger_ugsync_default_file, owner=params.unix_user, group=params.unix_group) if os.path.isfile(params.usgsync_log4j_file): File(params.usgsync_log4j_file, owner=params.unix_user, group=params.unix_group) if os.path.isfile(params.cred_validator_file): File(params.cred_validator_file, group=params.unix_group, mode=04555) ranger_credential_helper(params.ugsync_cred_lib, 'usersync.ssl.key.password', params.ranger_usersync_keystore_password, params.ugsync_jceks_path) if not is_empty(params.ranger_usersync_ldap_ldapbindpassword) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder': ranger_credential_helper(params.ugsync_cred_lib, 'ranger.usersync.ldap.bindalias', params.ranger_usersync_ldap_ldapbindpassword, params.ugsync_jceks_path) ranger_credential_helper(params.ugsync_cred_lib, 'usersync.ssl.truststore.password', params.ranger_usersync_truststore_password, params.ugsync_jceks_path) File(params.ugsync_jceks_path, owner = params.unix_user, group = params.unix_group, mode = 0640 ) File([params.usersync_start, params.usersync_stop], owner = params.unix_user, group = params.unix_group ) File(params.usersync_services_file, mode = 0755, ) if upgrade_type is not None and params.stack_supports_config_versioning: if os.path.islink('/usr/bin/ranger-usersync'): Link('/usr/bin/ranger-usersync', action="delete") Link('/usr/bin/ranger-usersync', to=params.usersync_services_file) Execute(('ln','-sf', format('{usersync_services_file}'),'/usr/bin/ranger-usersync'), not_if=format("ls /usr/bin/ranger-usersync"), only_if=format("ls {usersync_services_file}"), sudo=True) if not os.path.isfile(params.ranger_usersync_keystore_file): cmd = format("{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'") Execute(cmd, logoutput=True, user = params.unix_user) File(params.ranger_usersync_keystore_file, owner = params.unix_user, group = params.unix_group, mode = 0640 ) create_core_site_xml(ranger_ugsync_conf)
dfs_type=dfs_type) repo_name = str(config['clusterName']) + '_atlas' ssl_keystore_password = unicode( config['configurations']['ranger-atlas-policymgr-ssl'] ['xasecure.policymgr.clientssl.keystore.password']) ssl_truststore_password = unicode( config['configurations']['ranger-atlas-policymgr-ssl'] ['xasecure.policymgr.clientssl.truststore.password']) credential_file = format('/etc/ranger/{repo_name}/cred.jceks') xa_audit_hdfs_is_enabled = default( '/configurations/ranger-atlas-audit/xasecure.audit.destination.hdfs', False) enable_ranger_atlas = config['configurations'][ 'ranger-atlas-plugin-properties']['ranger-atlas-plugin-enabled'] enable_ranger_atlas = not is_empty( enable_ranger_atlas) and enable_ranger_atlas.lower() == 'yes' enable_ranger_hbase = config['configurations'][ 'ranger-hbase-plugin-properties']['ranger-hbase-plugin-enabled'] enable_ranger_hbase = not is_empty( enable_ranger_hbase) and enable_ranger_hbase.lower() == 'yes' policymgr_mgr_url = config['configurations']['admin-properties'][ 'policymgr_external_url'] downloaded_custom_connector = None driver_curl_source = None driver_curl_target = None ranger_env = config['configurations']['ranger-env'] ranger_plugin_properties = config['configurations'][ 'ranger-atlas-plugin-properties']
pig_tar_dest_file = webhcat_apps_dir + "/pig.tar.gz" hadoop_streaming_tar_source = hadoop_streaming_jars # this contains * sqoop_tar_source = sqoop_tar_file # this contains * hadoop_streaming_tar_dest_dir = webhcat_apps_dir sqoop_tar_dest_dir = webhcat_apps_dir tarballs_mode = 0755 execute_path = os.environ['PATH'] + os.pathsep + hive_bin + os.pathsep + hadoop_bin_dir hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName'] hive_jdbc_connection_url = config['configurations']['hive-site']['javax.jdo.option.ConnectionURL'] hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword'] hive_metastore_user_passwd = unicode(hive_metastore_user_passwd) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd hive_metastore_db_type = config['configurations']['hive-env']['hive_database_type'] #HACK Temporarily use dbType=azuredb while invoking schematool if hive_metastore_db_type == "mssql": hive_metastore_db_type = "azuredb" #users hive_user = config['configurations']['hive-env']['hive_user'] #JDBC driver jar name hive_jdbc_driver = config['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName'] jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' hive_database_name = config['configurations']['hive-env']['hive_database_name'] hive_database = config['configurations']['hive-env']['hive_database'] hive_use_existing_db = hive_database.startswith('Existing') # NOT SURE THAT IT'S A GOOD IDEA TO USE PATH TO CLASS IN DRIVER, MAYBE IT WILL BE BETTER TO USE DB TYPE.
mysql_port = config['configurations']['metron-env'][ 'metron_enrichment_db_port'] mysql_adduser_path = tmp_dir + "/addMysqlUser.sh" mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh" mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh" enrichment_hosts = default("/clusterHostInfo/enrichment_host", []) enrichment_host = enrichment_hosts[0] if len(enrichment_hosts) > 0 else None enrichment_metron_user = config['configurations']['metron-env'][ 'metron_enrichment_db_user'] enrichment_metron_user_passwd = config['configurations']['metron-env'][ 'metron_enrichment_db_password'] enrichment_metron_user_passwd = unicode( enrichment_metron_user_passwd) if not is_empty( enrichment_metron_user_passwd) else enrichment_metron_user_passwd mysql_process_name = status_params.mysql_process_name # create partial functions with common arguments for every HdfsResource call # to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code HdfsResource = functools.partial( HdfsResource, user=hdfs_user, hdfs_resource_ignore_file= "/var/lib/ambari-agent/data/.hdfs_resource_ignore", security_enabled=security_enabled, keytab=hdfs_user_keytab, kinit_path_local=kinit_path_local, hadoop_bin_dir=hadoop_bin_dir, hadoop_conf_dir=hadoop_conf_dir, principal_name=hdfs_principal_name,
def setup_usersync(upgrade_type=None): import params usersync_home = params.usersync_home ranger_home = params.ranger_home ranger_ugsync_conf = params.ranger_ugsync_conf if not is_empty( params.ranger_usersync_ldap_ldapbindpassword ) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder': password_validation(params.ranger_usersync_ldap_ldapbindpassword) Directory(params.ranger_pid_dir, mode=0750, owner=params.unix_user, group=params.unix_group) Directory(params.usersync_log_dir, owner=params.unix_user, group=params.unix_group) Directory(ranger_ugsync_conf, owner=params.unix_user, group=params.unix_group, recursive=True) if upgrade_type is not None: src_file = format( '{usersync_home}/conf.dist/ranger-ugsync-default.xml') dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) # Added by RAI src_file = format('{usersync_home}/conf.dist/log4j.xml') dst_file = format('{usersync_conf}/log4j.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) src_file = format('{usersync_home}/conf.dist/log4j.properties') dst_file = format('{usersync_conf}/log4j.properties') Execute(('cp', '-f', src_file, dst_file), sudo=True) # Added by RAI # Copies /etc/hadoop/conf.empty/core-site.xml to /etc/ranger/admin/conf/core-site.xml src_file = format('/etc/hadoop/conf.empty/core-site.xml') dst_file = format('/etc/ranger/admin/conf/core-site.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) # Added by RAI #makes 1) ln - sf / etc / ranger / admin / conf / usr / lib / ranger - admin / ews / webapp / WEB - INF / classes # 2) ln - sf / var / log / ranger // usr / lib / ranger - usersync / logs# Execute(('ln', '-sf', format('{ranger_conf}'), format('{ranger_home}/ews/webapp/WEB-INF/classes')), not_if=format("ls {ranger_home}/ews/webapp/WEB-INF/classes"), only_if=format("ls {ranger_conf}"), sudo=True) Execute(('ln', '-sf', format('{usersync_log_dir}'), format('{usersync_home}/logs')), not_if=format("ls {usersync_home}/logs"), only_if=format("ls {usersync_log_dir}"), sudo=True) XmlConfig( "ranger-ugsync-site.xml", conf_dir=ranger_ugsync_conf, configurations=params.config['configurations']['ranger-ugsync-site'], configuration_attributes=params.config['configuration_attributes'] ['ranger-ugsync-site'], owner=params.unix_user, group=params.unix_group, mode=0644) if os.path.isfile(params.ranger_ugsync_default_file): File(params.ranger_ugsync_default_file, owner=params.unix_user, group=params.unix_group) if os.path.isfile(params.usgsync_log4j_file): File(params.usgsync_log4j_file, owner=params.unix_user, group=params.unix_group) if os.path.isfile(params.cred_validator_file): File(params.cred_validator_file, group=params.unix_group, mode=04555) cred_file = format('{ranger_home}/ranger_credential_helper.py') if os.path.isfile(format('{usersync_home}/ranger_credential_helper.py')): cred_file = format('{usersync_home}/ranger_credential_helper.py') cred_lib = os.path.join(usersync_home, "lib", "*") cred_setup_prefix = (cred_file, '-l', cred_lib) cred_setup = cred_setup_prefix + ( '-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.key.password', '-v', PasswordString( params.ranger_usersync_keystore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = cred_setup_prefix + ( '-f', params.ugsync_jceks_path, '-k', 'ranger.usersync.ldap.bindalias', '-v', PasswordString( params.ranger_usersync_ldap_ldapbindpassword), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = cred_setup_prefix + ( '-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.truststore.password', '-v', PasswordString(params.ranger_usersync_truststore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) File(params.ugsync_jceks_path, owner=params.unix_user, group=params.unix_group, mode=0640) File([params.usersync_start, params.usersync_stop], owner=params.unix_user, group=params.unix_group) File( params.usersync_services_file, mode=0755, ) Execute(('ln', '-sf', format('{usersync_services_file}'), '/usr/bin/ranger-usersync'), not_if=format("ls /usr/bin/ranger-usersync"), only_if=format("ls {usersync_services_file}"), sudo=True) if not os.path.isfile(params.ranger_usersync_keystore_file): cmd = format( "{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'" ) Execute(cmd, logoutput=True, user=params.unix_user) File(params.ranger_usersync_keystore_file, owner=params.unix_user, group=params.unix_group, mode=0640)
def hive(name=None): import params if name == 'hiveserver2': # HDP 2.1.* or lower if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major, "2.2.0.0") < 0: params.HdfsResource(params.webhcat_apps_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=0755 ) # Create webhcat dirs. if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir: params.HdfsResource(params.hcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hcat_user, mode=params.hcat_hdfs_user_mode ) params.HdfsResource(params.webhcat_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.webhcat_user, mode=params.webhcat_hdfs_user_mode ) # ****** Begin Copy Tarballs ****** # ********************************* # HDP 2.2 or higher, copy mapreduce.tar.gz to HDFS if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major, '2.2') >= 0: copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped) # Always copy pig.tar.gz and hive.tar.gz using the appropriate mode. # This can use a different source and dest location to account for both HDP 2.1 and 2.2 copy_to_hdfs("pig", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.pig_tar_source, custom_dest_file=params.pig_tar_dest_file, host_sys_prepped=params.host_sys_prepped) copy_to_hdfs("hive", params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=params.hive_tar_source, custom_dest_file=params.hive_tar_dest_file, host_sys_prepped=params.host_sys_prepped) wildcard_tarballs = ["sqoop", "hadoop_streaming"] for tarball_name in wildcard_tarballs: source_file_pattern = eval("params." + tarball_name + "_tar_source") dest_dir = eval("params." + tarball_name + "_tar_dest_dir") if source_file_pattern is None or dest_dir is None: continue source_files = glob.glob(source_file_pattern) if "*" in source_file_pattern else [source_file_pattern] for source_file in source_files: src_filename = os.path.basename(source_file) dest_file = os.path.join(dest_dir, src_filename) copy_to_hdfs(tarball_name, params.user_group, params.hdfs_user, file_mode=params.tarballs_mode, custom_source_file=source_file, custom_dest_file=dest_file, host_sys_prepped=params.host_sys_prepped) # ******* End Copy Tarballs ******* # ********************************* # Create Hive Metastore Warehouse Dir params.HdfsResource(params.hive_apps_whs_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=0777 ) # Create Hive User Dir params.HdfsResource(params.hive_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.hive_user, mode=params.hive_hdfs_user_mode ) if not is_empty(params.hive_exec_scratchdir) and not urlparse(params.hive_exec_scratchdir).path.startswith("/tmp"): params.HdfsResource(params.hive_exec_scratchdir, type="directory", action="create_on_execute", owner=params.hive_user, group=params.hdfs_user, mode=0777) # Hive expects this dir to be writeable by everyone as it is used as a temp dir params.HdfsResource(None, action="execute") Directory(params.hive_etc_dir_prefix, mode=0755 ) # We should change configurations for client as well as for server. # The reason is that stale-configs are service-level, not component. for conf_dir in params.hive_conf_dirs_list: fill_conf_dir(conf_dir) XmlConfig("hive-site.xml", conf_dir=params.hive_config_dir, configurations=params.hive_site_config, configuration_attributes=params.config['configuration_attributes']['hive-site'], owner=params.hive_user, group=params.user_group, mode=0644) setup_atlas_hive() if params.hive_specific_configs_supported and name == 'hiveserver2': XmlConfig("hiveserver2-site.xml", conf_dir=params.hive_server_conf_dir, configurations=params.config['configurations']['hiveserver2-site'], configuration_attributes=params.config['configuration_attributes']['hiveserver2-site'], owner=params.hive_user, group=params.user_group, mode=0644) File(format("{hive_config_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, content=InlineTemplate(params.hive_env_sh_template) ) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, recursive=True, owner='root', group='root' ) File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, content=Template("hive.conf.j2") ) if (name == 'metastore' or name == 'hiveserver2') and not os.path.exists(params.target): jdbc_connector() File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), mode = 0644, ) if name == 'metastore': File(params.start_metastore_path, mode=0755, content=StaticFile('startMetastore.sh') ) if params.init_metastore_schema: create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_bin}/schematool -initSchema " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p}") check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; " "{hive_bin}/schematool -info " "-dbType {hive_metastore_db_type} " "-userName {hive_metastore_user_name} " "-passWord {hive_metastore_user_passwd!p}"), params.hive_user) # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords. # Fixing it with the hack below: quoted_hive_metastore_user_passwd = quote_bash_args(quote_bash_args(params.hive_metastore_user_passwd)) if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \ or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"': quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[1:-1] Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(check_schema_created_cmd.replace( format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING)) Execute(create_schema_cmd, not_if = check_schema_created_cmd, user = params.hive_user ) elif name == 'hiveserver2': File(params.start_hiveserver2_path, mode=0755, content=Template(format('{start_hiveserver2_script}')) ) if name != "client": crt_directory(params.hive_pid_dir) crt_directory(params.hive_log_dir) crt_directory(params.hive_var_lib)
def get_port_from_url(address): if not is_empty(address): return address.split(':')[-1] else: return address
hive_tar_dest_file = webhcat_apps_dir + "/hive.tar.gz" pig_tar_dest_file = webhcat_apps_dir + "/pig.tar.gz" hadoop_streaming_tar_source = hadoop_streaming_jars # this contains * sqoop_tar_source = sqoop_tar_file # this contains * hadoop_streaming_tar_dest_dir = webhcat_apps_dir sqoop_tar_dest_dir = webhcat_apps_dir tarballs_mode = 0755 execute_path = os.environ['PATH'] + os.pathsep + hive_bin + os.pathsep + hadoop_bin_dir hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName'] hive_jdbc_connection_url = config['configurations']['hive-site']['javax.jdo.option.ConnectionURL'] hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword'] hive_metastore_user_passwd = unicode(hive_metastore_user_passwd) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd hive_metastore_db_type = config['configurations']['hive-env']['hive_database_type'] #HACK Temporarily use dbType=azuredb while invoking schematool if hive_metastore_db_type == "mssql": hive_metastore_db_type = "azuredb" #users hive_user = config['configurations']['hive-env']['hive_user'] #JDBC driver jar name hive_jdbc_driver = config['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName'] # NOT SURE THAT IT'S A GOOD IDEA TO USE PATH TO CLASS IN DRIVER, MAYBE IT WILL BE BETTER TO USE DB TYPE. # BECAUSE PATH TO CLASSES COULD BE CHANGED sqla_db_used = False if hive_jdbc_driver == "com.microsoft.sqlserver.jdbc.SQLServerDriver": jdbc_jar_name = "sqljdbc4.jar" jdbc_symlink_name = "mssql-jdbc-driver.jar"
def metadata(type='server'): import params # Needed by both Server and Client Directory(params.conf_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) if type == "server": Directory([params.pid_dir], mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(format('{conf_dir}/solr'), mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True, recursive_ownership=True) Directory(params.log_dir, mode=0755, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.data_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) Directory(params.expanded_war_dir, mode=0644, cd_access='a', owner=params.metadata_user, group=params.user_group, create_parents=True) File(format("{expanded_war_dir}/atlas.war"), content=StaticFile( format('{metadata_home}/server/webapp/atlas.war'))) File(format("{conf_dir}/atlas-log4j.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_log4j_content)) File(format("{conf_dir}/atlas-env.sh"), owner=params.metadata_user, group=params.user_group, mode=0755, content=InlineTemplate(params.metadata_env_content)) if not is_empty(params.atlas_admin_username) and not is_empty( params.atlas_admin_password): psswd_output = hashlib.sha256( params.atlas_admin_password).hexdigest() ModifyPropertiesFile( format("{conf_dir}/users-credentials.properties"), properties={ format('{atlas_admin_username}'): format('ROLE_ADMIN::{psswd_output}') }, owner=params.metadata_user) files_to_chown = [ format("{conf_dir}/policy-store.txt"), format("{conf_dir}/users-credentials.properties") ] for file in files_to_chown: if os.path.exists(file): Execute( ('chown', format('{metadata_user}:{user_group}'), file), sudo=True) Execute(('chmod', '644', file), sudo=True) if params.metadata_solrconfig_content: File(format("{conf_dir}/solr/solrconfig.xml"), mode=0644, owner=params.metadata_user, group=params.user_group, content=InlineTemplate(params.metadata_solrconfig_content)) # Needed by both Server and Client PropertiesFile(format('{conf_dir}/{conf_file}'), properties=params.application_properties, mode=0644, owner=params.metadata_user, group=params.user_group) if params.security_enabled: TemplateConfig(format(params.atlas_jaas_file), owner=params.metadata_user) if type == 'server' and params.search_backend_solr and params.has_infra_solr: solr_cloud_util.setup_solr_client(params.config) check_znode() jaasFile = params.atlas_jaas_file if params.security_enabled else None upload_conf_set('atlas_configs', jaasFile) if params.security_enabled: # update permissions before creating the collections solr_cloud_util.add_solr_roles( params.config, roles=[ params.infra_solr_role_atlas, params.infra_solr_role_ranger_audit, params.infra_solr_role_dev ], new_service_principals=[params.atlas_jaas_principal]) create_collection('vertex_index', 'atlas_configs', jaasFile) create_collection('edge_index', 'atlas_configs', jaasFile) create_collection('fulltext_index', 'atlas_configs', jaasFile) if params.security_enabled: secure_znode(format('{infra_solr_znode}/configs/atlas_configs'), jaasFile) secure_znode(format('{infra_solr_znode}/collections/vertex_index'), jaasFile) secure_znode(format('{infra_solr_znode}/collections/edge_index'), jaasFile) secure_znode( format('{infra_solr_znode}/collections/fulltext_index'), jaasFile) File(params.atlas_hbase_setup, group=params.user_group, owner=params.hbase_user, content=Template("atlas_hbase_setup.rb.j2")) is_atlas_upgrade_support = check_stack_feature( StackFeature.ATLAS_UPGRADE_SUPPORT, get_stack_feature_version(params.config)) if is_atlas_upgrade_support and params.security_enabled: File(params.atlas_kafka_setup, group=params.user_group, owner=params.kafka_user, content=Template("atlas_kafka_acl.sh.j2")) # files required only in case if kafka broker is not present on the host as configured component if not params.host_with_kafka: File(format("{kafka_conf_dir}/kafka-env.sh"), owner=params.kafka_user, content=InlineTemplate(params.kafka_env_sh_template)) File(format("{kafka_conf_dir}/kafka_jaas.conf"), group=params.user_group, owner=params.kafka_user, content=Template("kafka_jaas.conf.j2")) if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len( params.namenode_host) > 1: XmlConfig( "hdfs-site.xml", conf_dir=params.conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params.config['configuration_attributes'] ['hdfs-site'], owner=params.metadata_user, group=params.user_group, mode=0644) else: File(format('{conf_dir}/hdfs-site.xml'), action="delete")
version_for_stack_feature_checks): hive_metastore_site_supported = True execute_path = os.environ[ 'PATH'] + os.pathsep + hive_bin + os.pathsep + hadoop_bin_dir hive_metastore_user_name = config['configurations']['hive-site'][ 'javax.jdo.option.ConnectionUserName'] hive_jdbc_connection_url = config['configurations']['hive-site'][ 'javax.jdo.option.ConnectionURL'] hive_metastore_user_passwd = config['configurations']['hive-site'][ 'javax.jdo.option.ConnectionPassword'] hive_metastore_user_passwd = unicode( hive_metastore_user_passwd ) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd hive_metastore_db_type = config['configurations']['hive-env'][ 'hive_database_type'] #HACK Temporarily use dbType=azuredb while invoking schematool if hive_metastore_db_type == "mssql": hive_metastore_db_type = "azuredb" #users hive_user = config['configurations']['hive-env']['hive_user'] #JDBC driver jar name hive_jdbc_driver = config['configurations']['hive-site'][ 'javax.jdo.option.ConnectionDriverName'] jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java'
# *********************** RANGER PLUGIN CHANGES *********************** # ranger host # ********************************************************************** ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) has_ranger_admin = not len(ranger_admin_hosts) == 0 xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] ranger_admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir","/var/log/ranger/admin") is_supported_kafka_ranger = config['configurations']['kafka-env']['is_supported_kafka_ranger'] #ranger kafka properties if has_ranger_admin and is_supported_kafka_ranger: enable_ranger_kafka = config['configurations']['ranger-kafka-plugin-properties']['ranger-kafka-plugin-enabled'] enable_ranger_kafka = not is_empty(enable_ranger_kafka) and enable_ranger_kafka.lower() == 'yes' policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] sql_connector_jar = config['configurations']['admin-properties']['SQL_CONNECTOR_JAR'] xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR'] xa_audit_db_flavor = xa_audit_db_flavor.lower() if xa_audit_db_flavor else None xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name'] xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user'] xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) xa_db_host = config['configurations']['admin-properties']['db_host'] repo_name = str(config['clusterName']) + '_kafka' ranger_env = config['configurations']['ranger-env'] ranger_plugin_properties = config['configurations']['ranger-kafka-plugin-properties'] ranger_kafka_audit = config['configurations']['ranger-kafka-audit'] ranger_kafka_audit_attrs = config['configuration_attributes']['ranger-kafka-audit']
security_enabled=security_enabled, keytab=hdfs_user_keytab, kinit_path_local=kinit_path_local, hadoop_bin_dir=hadoop_bin_dir, hadoop_conf_dir=hadoop_conf_dir, principal_name=hdfs_principal_name, hdfs_site=hdfs_site, default_fs=default_fs, immutable_paths=get_not_managed_resources(), dfs_type=dfs_type) # ranger atlas service/repository name repo_name = str(config['clusterName']) + '_atlas' repo_name_value = config['configurations']['ranger-atlas-security'][ 'ranger.plugin.atlas.service.name'] if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": repo_name = repo_name_value ssl_keystore_password = config['configurations'][ 'ranger-atlas-policymgr-ssl'][ 'xasecure.policymgr.clientssl.keystore.password'] ssl_truststore_password = config['configurations'][ 'ranger-atlas-policymgr-ssl'][ 'xasecure.policymgr.clientssl.truststore.password'] credential_file = format('/etc/ranger/{repo_name}/cred.jceks') xa_audit_hdfs_is_enabled = default( '/configurations/ranger-atlas-audit/xasecure.audit.destination.hdfs', False) # get ranger policy url policymgr_mgr_url = config['configurations']['ranger-atlas-security'][
db_name = config['configurations']['kms-properties']['db_name'] db_user = config['configurations']['kms-properties']['db_user'] db_password = unicode( config['configurations']['kms-properties']['db_password']) kms_master_key_password = unicode( config['configurations']['kms-properties']['KMS_MASTER_KEY_PASSWD']) credential_provider_path = config['configurations']['dbks-site'][ 'ranger.ks.jpa.jdbc.credential.provider.path'] jdbc_alias = config['configurations']['dbks-site'][ 'ranger.ks.jpa.jdbc.credential.alias'] masterkey_alias = config['configurations']['dbks-site'][ 'ranger.ks.masterkey.credential.alias'] repo_name = str(config['clusterName']) + '_kms' repo_name_value = config['configurations']['ranger-kms-security'][ 'ranger.plugin.kms.service.name'] if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": repo_name = repo_name_value cred_lib_path = os.path.join(kms_home, "cred", "lib", "*") cred_setup_prefix = (format('{kms_home}/ranger_credential_helper.py'), '-l', cred_lib_path) credential_file = format('/etc/ranger/{repo_name}/cred.jceks') # ranger kms ssl enabled config ranger_kms_ssl_enabled = config['configurations']['ranger-kms-site'][ 'ranger.service.https.attrib.ssl.enabled'] url_scheme = "http" if ranger_kms_ssl_enabled: url_scheme = "https" if has_ranger_admin: policymgr_mgr_url = config['configurations']['admin-properties'][
def setup_usersync(upgrade_type=None): import params usersync_home = params.usersync_home ranger_home = params.ranger_home ranger_ugsync_conf = params.ranger_ugsync_conf if not is_empty(params.ranger_usersync_ldap_ldapbindpassword) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder': password_validation(params.ranger_usersync_ldap_ldapbindpassword) if upgrade_type is not None: usersync_home = format("/usr/hdp/{version}/ranger-usersync") ranger_home = format("/usr/hdp/{version}/ranger-admin") ranger_ugsync_conf = format("/usr/hdp/{version}/ranger-usersync/conf") Directory(params.ranger_pid_dir, mode=0750, owner = params.unix_user, group = params.unix_group ) Directory(params.usersync_log_dir, owner = params.unix_user, group = params.unix_group ) Directory(format("{ranger_ugsync_conf}/"), owner = params.unix_user ) if upgrade_type is not None: src_file = format('{usersync_home}/conf.dist/ranger-ugsync-default.xml') dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) src_file = format('{usersync_home}/conf.dist/log4j.xml') dst_file = format('{usersync_home}/conf/log4j.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) XmlConfig("ranger-ugsync-site.xml", conf_dir=ranger_ugsync_conf, configurations=params.config['configurations']['ranger-ugsync-site'], configuration_attributes=params.config['configuration_attributes']['ranger-ugsync-site'], owner=params.unix_user, group=params.unix_group, mode=0644) if os.path.isfile(params.ranger_ugsync_default_file): File(params.ranger_ugsync_default_file, owner=params.unix_user, group=params.unix_group) if os.path.isfile(params.usgsync_log4j_file): File(params.usgsync_log4j_file, owner=params.unix_user, group=params.unix_group) if os.path.isfile(params.cred_validator_file): File(params.cred_validator_file, group=params.unix_group, mode=04555) cred_file = format('{ranger_home}/ranger_credential_helper.py') if os.path.isfile(format('{usersync_home}/ranger_credential_helper.py')): cred_file = format('{usersync_home}/ranger_credential_helper.py') cred_lib = os.path.join(usersync_home,"lib","*") cred_setup_prefix = (cred_file, '-l', cred_lib) cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.key.password', '-v', PasswordString(params.ranger_usersync_keystore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'ranger.usersync.ldap.bindalias', '-v', PasswordString(params.ranger_usersync_ldap_ldapbindpassword), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.truststore.password', '-v', PasswordString(params.ranger_usersync_truststore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) File(params.ugsync_jceks_path, owner = params.unix_user, group = params.unix_group, mode = 0640 ) File([params.usersync_start, params.usersync_stop], owner = params.unix_user, group = params.unix_group ) File(params.usersync_services_file, mode = 0755, ) Execute(('ln','-sf', format('{usersync_services_file}'),'/usr/bin/ranger-usersync'), not_if=format("ls /usr/bin/ranger-usersync"), only_if=format("ls {usersync_services_file}"), sudo=True) if not os.path.isfile(params.ranger_usersync_keystore_file): cmd = format("{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'") Execute(cmd, logoutput=True, user = params.unix_user) File(params.ranger_usersync_keystore_file, owner = params.unix_user, group = params.unix_group, mode = 0640 )