hive_metastore_site_supported = True execute_path = os.environ['PATH'] + os.pathsep + hive_bin + os.pathsep + hadoop_bin_dir hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName'] hive_jdbc_connection_url = config['configurations']['hive-site']['javax.jdo.option.ConnectionURL'] jdk_location = config['hostLevelParams']['jdk_location'] if credential_store_enabled: if 'hadoop.security.credential.provider.path' in config['configurations']['hive-site']: cs_lib_path = config['configurations']['hive-site']['credentialStoreClassPath'] java_home = config['hostLevelParams']['java_home'] alias = 'javax.jdo.option.ConnectionPassword' provider_path = config['configurations']['hive-site']['hadoop.security.credential.provider.path'] hive_metastore_user_passwd = PasswordString(get_password_from_credential_store(alias, provider_path, cs_lib_path, java_home, jdk_location)) else: raise Exception("hadoop.security.credential.provider.path property should be set") else: hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword'] hive_metastore_user_passwd = unicode(hive_metastore_user_passwd) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd hive_metastore_db_type = config['configurations']['hive-env']['hive_database_type'] #HACK Temporarily use dbType=azuredb while invoking schematool if hive_metastore_db_type == "mssql": hive_metastore_db_type = "azuredb" #users hive_user = config['configurations']['hive-env']['hive_user'] # is it a restart command
def ranger_credential_helper(lib_path, alias_key, alias_value, file_path): import params java_bin = format('{java_home}/bin/java') file_path = format('jceks://file{file_path}') cmd = (java_bin, '-cp', lib_path, 'org.apache.ranger.credentialapi.buildks', 'create', alias_key, '-value', PasswordString(alias_value), '-provider', file_path) Execute(cmd, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, hdp_version, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, component_user, component_group, java_home): cred_lib_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install/lib/*') cred_setup_prefix = (format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/ranger_credential_helper.py'), '-l', cred_lib_path) if audit_db_is_enabled: cred_setup = cred_setup_prefix + ('-f', credential_file, '-k', 'auditDBCred', '-v', PasswordString(xa_audit_db_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True, sudo=True) cred_setup = cred_setup_prefix + ('-f', credential_file, '-k', 'sslKeyStore', '-v', PasswordString(ssl_keystore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True, sudo=True) cred_setup = cred_setup_prefix + ('-f', credential_file, '-k', 'sslTrustStore', '-v', PasswordString(ssl_truststore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True, sudo=True) File(credential_file, owner = component_user, group = component_group, mode = 0640 )
def enable_kms_plugin(): import params if params.has_ranger_admin: ranger_flag = False if params.stack_supports_ranger_kerberos and params.security_enabled: if not is_empty(params.rangerkms_principal) and params.rangerkms_principal != '': ranger_flag = check_ranger_service_support_kerberos(params.kms_user, params.rangerkms_keytab, params.rangerkms_principal) else: ranger_flag = check_ranger_service_support_kerberos(params.kms_user, params.spengo_keytab, params.spnego_principal) else: ranger_flag = check_ranger_service() if not ranger_flag: Logger.error('Error in Get/Create service for Ranger Kms.') current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S") File(format('{kms_conf_dir}/ranger-security.xml'), owner = params.kms_user, group = params.kms_group, mode = 0644, content = format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>') ) Directory([os.path.join('/etc', 'ranger', params.repo_name), os.path.join('/etc', 'ranger', params.repo_name, 'policycache')], owner = params.kms_user, group = params.kms_group, mode=0775, create_parents = True ) File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',format('kms_{repo_name}.json')), owner = params.kms_user, group = params.kms_group, mode = 0644 ) # remove plain-text password from xml configs plugin_audit_properties_copy = {} plugin_audit_properties_copy.update(params.config['configurations']['ranger-kms-audit']) if params.plugin_audit_password_property in plugin_audit_properties_copy: plugin_audit_properties_copy[params.plugin_audit_password_property] = "crypted" XmlConfig("ranger-kms-audit.xml", conf_dir=params.kms_conf_dir, configurations=plugin_audit_properties_copy, configuration_attributes=params.config['configuration_attributes']['ranger-kms-audit'], owner=params.kms_user, group=params.kms_group, mode=0744) XmlConfig("ranger-kms-security.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations']['ranger-kms-security'], configuration_attributes=params.config['configuration_attributes']['ranger-kms-security'], owner=params.kms_user, group=params.kms_group, mode=0744) # remove plain-text password from xml configs ranger_kms_policymgr_ssl_copy = {} ranger_kms_policymgr_ssl_copy.update(params.config['configurations']['ranger-kms-policymgr-ssl']) for prop in params.kms_plugin_password_properties: if prop in ranger_kms_policymgr_ssl_copy: ranger_kms_policymgr_ssl_copy[prop] = "crypted" XmlConfig("ranger-policymgr-ssl.xml", conf_dir=params.kms_conf_dir, configurations=ranger_kms_policymgr_ssl_copy, configuration_attributes=params.config['configuration_attributes']['ranger-kms-policymgr-ssl'], owner=params.kms_user, group=params.kms_group, mode=0744) if params.xa_audit_db_is_enabled: cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'auditDBCred', '-v', PasswordString(params.xa_audit_db_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslKeyStore', '-v', PasswordString(params.ssl_keystore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslTrustStore', '-v', PasswordString(params.ssl_truststore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) File(params.credential_file, owner = params.kms_user, group = params.kms_group, mode = 0640 ) # create ranger kms audit directory if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node: params.HdfsResource("/ranger/audit", type="directory", action="create_on_execute", owner=params.hdfs_user, group=params.hdfs_user, mode=0755, recursive_chmod=True ) params.HdfsResource("/ranger/audit/kms", type="directory", action="create_on_execute", owner=params.kms_user, group=params.kms_group, mode=0750, recursive_chmod=True ) params.HdfsResource(None, action="execute") if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1: Logger.info('Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml') XmlConfig("hdfs-site.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations']['hdfs-site'], configuration_attributes=params.config['configuration_attributes']['hdfs-site'], owner=params.kms_user, group=params.kms_group, mode=0644 ) else: File(format('{kms_conf_dir}/hdfs-site.xml'), action="delete")
def enable_kms_plugin(): import params if params.has_ranger_admin: if params.stack_supports_ranger_kerberos and params.security_enabled: ranger_flag = check_ranger_service_support_kerberos() else: ranger_flag = check_ranger_service() if not ranger_flag: Logger.error('Error in Get/Create service for Ranger Kms.') current_datetime = datetime.now() File( format('{kms_conf_dir}/ranger-security.xml'), owner=params.kms_user, group=params.kms_group, mode=0644, content=InlineTemplate( format( '<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>' ))) Directory([ os.path.join('/etc', 'ranger', params.repo_name), os.path.join('/etc', 'ranger', params.repo_name, 'policycache') ], owner=params.kms_user, group=params.kms_group, mode=0775, create_parents=True) File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache', format('kms_{repo_name}.json')), owner=params.kms_user, group=params.kms_group, mode=0644) XmlConfig( "ranger-kms-audit.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations']['ranger-kms-audit'], configuration_attributes=params.config['configuration_attributes'] ['ranger-kms-audit'], owner=params.kms_user, group=params.kms_group, mode=0744) XmlConfig( "ranger-kms-security.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations'] ['ranger-kms-security'], configuration_attributes=params.config['configuration_attributes'] ['ranger-kms-security'], owner=params.kms_user, group=params.kms_group, mode=0744) XmlConfig( "ranger-policymgr-ssl.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations'] ['ranger-kms-policymgr-ssl'], configuration_attributes=params.config['configuration_attributes'] ['ranger-kms-policymgr-ssl'], owner=params.kms_user, group=params.kms_group, mode=0744) if params.xa_audit_db_is_enabled: cred_setup = params.cred_setup_prefix + ( '-f', params.credential_file, '-k', 'auditDBCred', '-v', PasswordString(params.xa_audit_db_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = params.cred_setup_prefix + ( '-f', params.credential_file, '-k', 'sslKeyStore', '-v', PasswordString(params.ssl_keystore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = params.cred_setup_prefix + ( '-f', params.credential_file, '-k', 'sslTrustStore', '-v', PasswordString(params.ssl_truststore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) File(params.credential_file, owner=params.kms_user, group=params.kms_group, mode=0640)
def do_keystore_setup(cred_provider_path, credential_alias, credential_password): import params if cred_provider_path is not None: cred_setup = params.cred_setup_prefix + ('-f', cred_provider_path, '-k', credential_alias, '-v', PasswordString(credential_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True, ) File(cred_provider_path, owner = params.kms_user, group = params.kms_group, mode = 0640 )
def encrypt_sensitive_properties( nifi_config_dir, jdk64_home, java_options, nifi_user, last_master_key, master_key_password, is_starting, toolkit_tmp_dir, stack_version_buildnum, service, nifi_flow_config_dir=None, nifi_sensitive_props_key=None, support_encrypt_authorizers=None, ): encrypt_config_script = get_toolkit_script('encrypt-config.sh', toolkit_tmp_dir, stack_version_buildnum) encrypt_config_command = (encrypt_config_script, ) environment = {'JAVA_HOME': jdk64_home, 'JAVA_OPTS': java_options} File(encrypt_config_script, mode=0755) if is_starting: if service == NIFI: Logger.info("Encrypting NiFi sensitive configuration properties") encrypt_config_command += ('-v', '-b', nifi_config_dir + '/bootstrap.conf') encrypt_config_command += ('-n', nifi_config_dir + '/nifi.properties') if (sudo.path_isfile(nifi_flow_config_dir + '/flow.xml.gz') and len(sudo.read_file(nifi_flow_config_dir + '/flow.xml.gz')) > 0): encrypt_config_command += ( '-f', nifi_flow_config_dir + '/flow.xml.gz', '-s', PasswordString(nifi_sensitive_props_key)) if contains_providers( nifi_config_dir + '/login-identity-providers.xml', "provider"): encrypt_config_command += ('-l', nifi_config_dir + '/login-identity-providers.xml') if support_encrypt_authorizers and contains_providers( nifi_config_dir + '/authorizers.xml', "authorizer"): encrypt_config_command += ('-a', nifi_config_dir + '/authorizers.xml') if last_master_key: encrypt_config_command += ('-m', '-e', PasswordString(last_master_key)) encrypt_config_command += ('-p', PasswordString(master_key_password)) Execute(encrypt_config_command, user=nifi_user, logoutput=False, environment=environment) elif service == NIFI_REGISTRY: Logger.info( "Encrypting NiFi Registry sensitive configuration properties") encrypt_config_command += ('--nifiRegistry', '-v', '-b', nifi_config_dir + '/bootstrap.conf') encrypt_config_command += ('-r', nifi_config_dir + '/nifi-registry.properties') if contains_providers(nifi_config_dir + '/identity-providers.xml', "provider"): encrypt_config_command += ('-i', nifi_config_dir + '/identity-providers.xml') if contains_providers(nifi_config_dir + '/authorizers.xml', "authorizer"): encrypt_config_command += ('-a', nifi_config_dir + '/authorizers.xml') if last_master_key: encrypt_config_command += ('--oldKey', PasswordString(last_master_key)) encrypt_config_command += ('-p', PasswordString(master_key_password)) Execute(encrypt_config_command, user=nifi_user, logoutput=False, environment=environment)
cmd = (java_bin, '-cp', cs_lib_path, credential_util_cmd, 'get', alias, '-provider', provider_path) cmd_result, std_out_msg = checked_call(cmd) if cmd_result != 0: message = 'The following error occurred while executing {0}: {1}'.format( ' '.join(cmd), std_out_msg) Logger.error(message) raise std_out_lines = std_out_msg.split('\n') passwd = std_out_lines[ -1] # Get the last line of the output, to skip warnings if any. return passwd if credential_store_enabled: hive_metastore_user_passwd = PasswordString(getHiveMetastorePassword()) else: hive_metastore_user_passwd = config['configurations']['hive-site'][ 'javax.jdo.option.ConnectionPassword'] hive_metastore_user_passwd = unicode( hive_metastore_user_passwd ) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd hive_metastore_db_type = config['configurations']['hive-env'][ 'hive_database_type'] #HACK Temporarily use dbType=azuredb while invoking schematool if hive_metastore_db_type == "mssql": hive_metastore_db_type = "azuredb" #users hive_user = config['configurations']['hive-env']['hive_user']
def do_keystore_setup(rolling_upgrade=False): import params ranger_home = params.ranger_home cred_lib_path = params.cred_lib_path cred_setup_prefix = params.cred_setup_prefix if rolling_upgrade: ranger_home = format("/usr/hdp/{version}/ranger-admin") cred_lib_path = os.path.join(ranger_home,"cred","lib","*") cred_setup_prefix = (format('{ranger_home}/ranger_credential_helper.py'), '-l', cred_lib_path) if not is_empty(params.ranger_credential_provider_path): jceks_path = params.ranger_credential_provider_path cred_setup = cred_setup_prefix + ('-f', jceks_path, '-k', params.ranger_jpa_jdbc_credential_alias, '-v', PasswordString(params.ranger_ambari_db_password), '-c', '1') Execute(cred_setup, environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True ) File(params.ranger_credential_provider_path, owner = params.unix_user, group = params.unix_group, mode = 0640 ) if not is_empty(params.ranger_credential_provider_path) and (params.ranger_audit_source_type).lower() == 'db' and not is_empty(params.ranger_ambari_audit_db_password): jceks_path = params.ranger_credential_provider_path cred_setup = cred_setup_prefix + ('-f', jceks_path, '-k', params.ranger_jpa_audit_jdbc_credential_alias, '-v', PasswordString(params.ranger_ambari_audit_db_password), '-c', '1') Execute(cred_setup, environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True ) File(params.ranger_credential_provider_path, owner = params.unix_user, group = params.unix_group, mode = 0640 )
def enable_kms_plugin(): import params if params.has_ranger_admin: ranger_adm_obj = Rangeradmin(url=params.policymgr_mgr_url) response_code, response_recieved = ranger_adm_obj.check_ranger_login_urllib2( params.policymgr_mgr_url + '/login.jsp', 'test:test') if response_code is not None and response_code == 200: ambari_ranger_admin, ambari_ranger_password = ranger_adm_obj.create_ambari_admin_user( params.ambari_ranger_admin, params.ambari_ranger_password, params.admin_uname_password) ambari_username_password_for_ranger = ambari_ranger_admin + ':' + ambari_ranger_password else: raise Fail('Ranger service is not started on given host') if ambari_ranger_admin != '' and ambari_ranger_password != '': get_repo_flag = get_repo(params.policymgr_mgr_url, params.repo_name, ambari_username_password_for_ranger) if not get_repo_flag: create_repo(params.policymgr_mgr_url, json.dumps(params.kms_ranger_plugin_repo), ambari_username_password_for_ranger) else: raise Fail('Ambari admin username and password not available') current_datetime = datetime.now() File( format('{kms_conf_dir}/ranger-security.xml'), owner=params.kms_user, group=params.kms_group, mode=0644, content=InlineTemplate( format( '<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>' ))) Directory([ os.path.join('/etc', 'ranger', params.repo_name), os.path.join('/etc', 'ranger', params.repo_name, 'policycache') ], owner=params.kms_user, group=params.kms_group, mode=0775, recursive=True) File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache', format('kms_{repo_name}.json')), owner=params.kms_user, group=params.kms_group, mode=0644) XmlConfig( "ranger-kms-audit.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations']['ranger-kms-audit'], configuration_attributes=params.config['configuration_attributes'] ['ranger-kms-audit'], owner=params.kms_user, group=params.kms_group, mode=0744) XmlConfig( "ranger-kms-security.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations'] ['ranger-kms-security'], configuration_attributes=params.config['configuration_attributes'] ['ranger-kms-security'], owner=params.kms_user, group=params.kms_group, mode=0744) XmlConfig( "ranger-policymgr-ssl.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations'] ['ranger-kms-policymgr-ssl'], configuration_attributes=params.config['configuration_attributes'] ['ranger-kms-policymgr-ssl'], owner=params.kms_user, group=params.kms_group, mode=0744) if params.xa_audit_db_is_enabled: cred_setup = params.cred_setup_prefix + ( '-f', params.credential_file, '-k', 'auditDBCred', '-v', PasswordString(params.xa_audit_db_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = params.cred_setup_prefix + ( '-f', params.credential_file, '-k', 'sslKeyStore', '-v', PasswordString(params.ssl_keystore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = params.cred_setup_prefix + ( '-f', params.credential_file, '-k', 'sslTrustStore', '-v', PasswordString(params.ssl_truststore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) File(params.credential_file, owner=params.kms_user, group=params.kms_group, mode=0640)
def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, stack_version, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, component_user, component_group, java_home, cred_lib_path_override = None, cred_setup_prefix_override = None): stack_root = Script.get_stack_root() service_name = str(service_name).lower() if cred_lib_path_override is not None: cred_lib_path = cred_lib_path_override else: cred_lib_path = format('{stack_root}/{stack_version}/ranger-{service_name}-plugin/install/lib/*') if cred_setup_prefix_override is not None: cred_setup_prefix = cred_setup_prefix_override else: cred_setup_prefix = (format('{stack_root}/{stack_version}/ranger-{service_name}-plugin/ranger_credential_helper.py'), '-l', cred_lib_path) if audit_db_is_enabled: cred_setup = cred_setup_prefix + ('-f', credential_file, '-k', 'auditDBCred', '-v', PasswordString(xa_audit_db_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True, sudo=True) cred_setup = cred_setup_prefix + ('-f', credential_file, '-k', 'sslKeyStore', '-v', PasswordString(ssl_keystore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True, sudo=True) cred_setup = cred_setup_prefix + ('-f', credential_file, '-k', 'sslTrustStore', '-v', PasswordString(ssl_truststore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True, sudo=True) File(credential_file, owner = component_user, group = component_group, mode = 0640 )
def log_str(self, key, value): # Hide the passwords from text representations return repr(PasswordString(value))
def create_password_in_credential_store(alias, provider_path, cs_lib_path, java_home, jdk_location, password): downloadjar(cs_lib_path, jdk_location) #Execute the creation and overwrite password java_bin = '{java_home}/bin/java'.format(java_home=java_home) cmd = (java_bin, '-cp', cs_lib_path, credential_util_cmd, 'create', alias, '-value', PasswordString(password) ,'-provider', provider_path, '-f') Execute(cmd)
def setup_tagsync(upgrade_type=None): import params ranger_tagsync_home = params.ranger_tagsync_home ranger_home = params.ranger_home ranger_tagsync_conf = params.ranger_tagsync_conf tagsync_log4j_file = format('{ranger_tagsync_conf}/log4j.xml') tagsync_services_file = format( '{ranger_tagsync_home}/ranger-tagsync-services.sh') Directory(format("{ranger_tagsync_conf}"), owner=params.unix_user, group=params.unix_group, create_parents=True) Directory( params.ranger_pid_dir, mode=0750, create_parents=True, owner=params.unix_user, group=params.unix_group, cd_access="a", ) Directory(params.tagsync_log_dir, create_parents=True, owner=params.unix_user, group=params.unix_group, cd_access="a", mode=0755) File(format('{ranger_tagsync_conf}/ranger-tagsync-env-logdir.sh'), content=format("export RANGER_TAGSYNC_LOG_DIR={tagsync_log_dir}"), owner=params.unix_user, group=params.unix_group, mode=0755) XmlConfig( "ranger-tagsync-site.xml", conf_dir=ranger_tagsync_conf, configurations=params.config['configurations']['ranger-tagsync-site'], configuration_attributes=params.config['configuration_attributes'] ['ranger-tagsync-site'], owner=params.unix_user, group=params.unix_group, mode=0644) PropertiesFile(format('{ranger_tagsync_conf}/application.properties'), properties=params.tagsync_application_properties, mode=0755, owner=params.unix_user, group=params.unix_group) if upgrade_type is not None: src_file = format( '{ranger_tagsync_home}/ews/webapp/WEB-INF/classes/conf.dist/log4j.xml' ) dst_file = format('{tagsync_log4j_file}') Execute(('cp', '-f', src_file, dst_file), sudo=True) if os.path.isfile(tagsync_log4j_file): File(tagsync_log4j_file, owner=params.unix_user, group=params.unix_group) else: Logger.warning( 'Required file {0} does not exist, copying the file to {1} path'. format(tagsync_log4j_file, ranger_tagsync_conf)) src_file = format( '{ranger_tagsync_home}/ews/webapp/WEB-INF/classes/conf.dist/log4j.xml' ) dst_file = format('{tagsync_log4j_file}') Execute(('cp', '-f', src_file, dst_file), sudo=True) File(tagsync_log4j_file, owner=params.unix_user, group=params.unix_group) cred_file = format('{ranger_home}/ranger_credential_helper.py') if os.path.isfile( format('{ranger_tagsync_home}/ranger_credential_helper.py')): cred_file = format('{ranger_tagsync_home}/ranger_credential_helper.py') cred_lib = os.path.join(ranger_tagsync_home, "lib", "*") cred_setup_prefix = (cred_file, '-l', cred_lib) if not is_empty(params.tagsync_jceks_path) and not is_empty( params.ranger_tagsync_tagadmin_password ) and params.tagsync_enabled: cred_setup = cred_setup_prefix + ( '-f', params.tagsync_jceks_path, '-k', 'tagadmin.user.password', '-v', PasswordString( params.ranger_tagsync_tagadmin_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) File(params.tagsync_jceks_path, owner=params.unix_user, group=params.unix_group, mode=0640) if os.path.isfile(tagsync_services_file): File( tagsync_services_file, mode=0755, ) Execute(('ln', '-sf', format('{tagsync_services_file}'), '/usr/bin/ranger-tagsync'), not_if=format("ls /usr/bin/ranger-tagsync"), only_if=format("ls {tagsync_services_file}"), sudo=True)
def setup_usersync(upgrade_type=None): import params usersync_home = params.usersync_home ranger_home = params.ranger_home ranger_ugsync_conf = params.ranger_ugsync_conf if not is_empty( params.ranger_usersync_ldap_ldapbindpassword ) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder': password_validation(params.ranger_usersync_ldap_ldapbindpassword) if upgrade_type is not None: usersync_home = format("/usr/hdp/{version}/ranger-usersync") ranger_home = format("/usr/hdp/{version}/ranger-admin") ranger_ugsync_conf = format("/usr/hdp/{version}/ranger-usersync/conf") Directory(params.ranger_pid_dir, mode=0750, owner=params.unix_user, group=params.unix_group) Directory(params.usersync_log_dir, owner=params.unix_user, group=params.unix_group) Directory(format("{ranger_ugsync_conf}/"), owner=params.unix_user) if upgrade_type is not None: src_file = format( '{usersync_home}/conf.dist/ranger-ugsync-default.xml') dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) src_file = format('{usersync_home}/conf.dist/log4j.xml') dst_file = format('{usersync_home}/conf/log4j.xml') Execute(('cp', '-f', src_file, dst_file), sudo=True) XmlConfig( "ranger-ugsync-site.xml", conf_dir=ranger_ugsync_conf, configurations=params.config['configurations']['ranger-ugsync-site'], configuration_attributes=params.config['configuration_attributes'] ['ranger-ugsync-site'], owner=params.unix_user, group=params.unix_group, mode=0644) if os.path.isfile(params.ranger_ugsync_default_file): File(params.ranger_ugsync_default_file, owner=params.unix_user, group=params.unix_group) if os.path.isfile(params.usgsync_log4j_file): File(params.usgsync_log4j_file, owner=params.unix_user, group=params.unix_group) if os.path.isfile(params.cred_validator_file): File(params.cred_validator_file, group=params.unix_group, mode=04555) cred_file = format('{ranger_home}/ranger_credential_helper.py') if os.path.isfile(format('{usersync_home}/ranger_credential_helper.py')): cred_file = format('{usersync_home}/ranger_credential_helper.py') cred_lib = os.path.join(usersync_home, "lib", "*") cred_setup_prefix = (cred_file, '-l', cred_lib) cred_setup = cred_setup_prefix + ( '-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.key.password', '-v', PasswordString( params.ranger_usersync_keystore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = cred_setup_prefix + ( '-f', params.ugsync_jceks_path, '-k', 'ranger.usersync.ldap.bindalias', '-v', PasswordString( params.ranger_usersync_ldap_ldapbindpassword), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = cred_setup_prefix + ( '-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.truststore.password', '-v', PasswordString(params.ranger_usersync_truststore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) File(params.ugsync_jceks_path, owner=params.unix_user, group=params.unix_group, mode=0640) File([params.usersync_start, params.usersync_stop], owner=params.unix_user, group=params.unix_group) File( params.usersync_services_file, mode=0755, ) Execute(('ln', '-sf', format('{usersync_services_file}'), '/usr/bin/ranger-usersync'), not_if=format("ls /usr/bin/ranger-usersync"), only_if=format("ls {usersync_services_file}"), sudo=True) if not os.path.isfile(params.ranger_usersync_keystore_file): cmd = format( "{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'" ) Execute(cmd, logoutput=True, user=params.unix_user) File(params.ranger_usersync_keystore_file, owner=params.unix_user, group=params.unix_group, mode=0640)
def setup_usersync(): import params Directory(params.ranger_pid_dir, mode=0750, owner = params.unix_user, group = params.unix_group ) Directory(params.usersync_log_dir, owner = params.unix_user, group = params.unix_group ) Directory(format("{ranger_ugsync_conf}/"), owner = params.unix_user ) XmlConfig("ranger-ugsync-site.xml", conf_dir=params.ranger_ugsync_conf, configurations=params.config['configurations']['ranger-ugsync-site'], configuration_attributes=params.config['configuration_attributes']['ranger-ugsync-site'], owner=params.unix_user, group=params.unix_group, mode=0644) cred_lib = os.path.join(params.usersync_home,"lib","*") cred_setup_prefix = (format('{ranger_home}/ranger_credential_helper.py'), '-l', cred_lib) cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.key.password', '-v', PasswordString(params.ranger_usersync_keystore_password), '-c', '1') Execute(cred_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'ranger.usersync.ldap.bindalias', '-v', PasswordString(params.ranger_usersync_ldap_ldapbindpassword), '-c', '1') Execute(cred_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.truststore.password', '-v', PasswordString(params.ranger_usersync_truststore_password), '-c', '1') Execute(cred_setup, environment={'RANGER_ADMIN_HOME':params.ranger_home, 'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) File(params.ugsync_jceks_path, owner = params.unix_user, group = params.unix_group, mode = 0640 ) File([params.usersync_start, params.usersync_stop], owner = params.unix_user, group = params.unix_group ) File(params.usersync_services_file, mode = 0755, ) Execute(('ln','-sf', format('{usersync_services_file}'),'/usr/bin/ranger-usersync'), not_if=format("ls /usr/bin/ranger-usersync"), only_if=format("ls {usersync_services_file}"), sudo=True) if not os.path.isfile(params.ranger_usersync_keystore_file): cmd = format("{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'") Execute(cmd, logoutput=True, user = params.unix_user) File(params.ranger_usersync_keystore_file, owner = params.unix_user, group = params.unix_group, mode = 0640 )
def generateJceks(self, commandJson): """ Generates the JCEKS file with passwords for the service specified in commandJson :param commandJson: command JSON :return: An exit value from the external process that generated the JCEKS file. None if there are no passwords in the JSON. """ cmd_result = None roleCommand = None if 'roleCommand' in commandJson: roleCommand = commandJson['roleCommand'] task_id = None if 'taskId' in commandJson: task_id = commandJson['taskId'] logger.info( 'Generating the JCEKS file: roleCommand={0} and taskId = {1}'. format(roleCommand, task_id)) # Set up the variables for the external command to generate a JCEKS file java_home = commandJson['hostLevelParams']['java_home'] java_bin = '{java_home}/bin/java'.format(java_home=java_home) cs_lib_path = self.credential_shell_lib_path serviceName = commandJson['serviceName'] # Gather the password values and remove them from the configuration configtype_credentials = self.getConfigTypeCredentials(commandJson) # CS is enabled but no config property is available for this command if len(configtype_credentials) == 0: logger.info( "Credential store is enabled but no property are found that can be encrypted." ) commandJson['credentialStoreEnabled'] = "false" for config_type, credentials in configtype_credentials.items(): config = commandJson['configurations'][config_type] file_path = os.path.join(self.getProviderDirectory(serviceName), "{0}.jceks".format(config_type)) if os.path.exists(file_path): os.remove(file_path) provider_path = 'jceks://file{file_path}'.format( file_path=file_path) logger.info('provider_path={0}'.format(provider_path)) for alias, pwd in credentials.items(): logger.debug("config={0}".format(config)) protected_pwd = PasswordString(pwd) # Generate the JCEKS file cmd = (java_bin, '-cp', cs_lib_path, self.credential_shell_cmd, 'create', alias, '-value', protected_pwd, '-provider', provider_path) logger.info(cmd) cmd_result = subprocess.call(cmd) logger.info('cmd_result = {0}'.format(cmd_result)) os.chmod( file_path, 0644 ) # group and others should have read access so that the service user can read # Add JCEKS provider path instead config[self.CREDENTIAL_PROVIDER_PROPERTY_NAME] = provider_path config[self.CREDENTIAL_STORE_CLASS_PATH_NAME] = cs_lib_path return cmd_result
def do_keystore_setup(cred_provider_path, credential_alias, credential_password): import params if cred_provider_path is not None: java_bin = format('{java_home}/bin/java') file_path = format('jceks://file{cred_provider_path}') cmd = (java_bin, '-cp', params.cred_lib_path, 'org.apache.ranger.credentialapi.buildks', 'create', credential_alias, '-value', PasswordString(credential_password), '-provider', file_path) Execute(cmd, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True, ) File(cred_provider_path, owner = params.kms_user, group = params.kms_group, mode = 0640 )
def enable_kms_plugin(): import params if params.has_ranger_admin: count = 0 while count < 5: ranger_flag = check_ranger_service() if ranger_flag: break else: time.sleep(5) # delay for 5 seconds count = count + 1 else: Logger.error("Ranger service is not reachable after {0} tries".format(count)) current_datetime = datetime.now() File(format('{kms_conf_dir}/ranger-security.xml'), owner = params.kms_user, group = params.kms_group, mode = 0644, content = InlineTemplate(format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>')) ) Directory([os.path.join('/etc', 'ranger', params.repo_name), os.path.join('/etc', 'ranger', params.repo_name, 'policycache')], owner = params.kms_user, group = params.kms_group, mode=0775, recursive = True ) File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',format('kms_{repo_name}.json')), owner = params.kms_user, group = params.kms_group, mode = 0644 ) XmlConfig("ranger-kms-audit.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations']['ranger-kms-audit'], configuration_attributes=params.config['configuration_attributes']['ranger-kms-audit'], owner=params.kms_user, group=params.kms_group, mode=0744) XmlConfig("ranger-kms-security.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations']['ranger-kms-security'], configuration_attributes=params.config['configuration_attributes']['ranger-kms-security'], owner=params.kms_user, group=params.kms_group, mode=0744) XmlConfig("ranger-policymgr-ssl.xml", conf_dir=params.kms_conf_dir, configurations=params.config['configurations']['ranger-kms-policymgr-ssl'], configuration_attributes=params.config['configuration_attributes']['ranger-kms-policymgr-ssl'], owner=params.kms_user, group=params.kms_group, mode=0744) if params.xa_audit_db_is_enabled: cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'auditDBCred', '-v', PasswordString(params.xa_audit_db_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslKeyStore', '-v', PasswordString(params.ssl_keystore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) cred_setup = params.cred_setup_prefix + ('-f', params.credential_file, '-k', 'sslTrustStore', '-v', PasswordString(params.ssl_truststore_password), '-c', '1') Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True) File(params.credential_file, owner = params.kms_user, group = params.kms_group, mode = 0640 )
def encrypt_sensitive_properties(config_version_file, current_version, nifi_config_dir, jdk64_home, java_options, nifi_user, nifi_group, master_key_password, nifi_flow_config_dir, nifi_sensitive_props_key, is_starting, toolkit_tmp_dir, support_encrypt_authorizers): Logger.info("Encrypting NiFi sensitive configuration properties") encrypt_config_script = get_toolkit_script('encrypt-config.sh', toolkit_tmp_dir) encrypt_config_command = (encrypt_config_script, ) environment = {'JAVA_HOME': jdk64_home} if java_options: environment['JAVA_OPTS'] = java_options File(encrypt_config_script, mode=0755) if is_starting: last_master_key_password = None last_config_version = get_config_version(config_version_file, 'encrypt') encrypt_config_command += ('-v', '-b', nifi_config_dir + '/bootstrap.conf') encrypt_config_command += ('-n', nifi_config_dir + '/nifi.properties') if (sudo.path_isfile(nifi_flow_config_dir + '/flow.xml.gz') and len( sudo.read_file(nifi_flow_config_dir + '/flow.xml.gz')) > 0): encrypt_config_command += ( '-f', nifi_flow_config_dir + '/flow.xml.gz', '-s', PasswordString(nifi_sensitive_props_key)) if contains_providers( nifi_config_dir + '/login-identity-providers.xml', "provider"): encrypt_config_command += ('-l', nifi_config_dir + '/login-identity-providers.xml') if support_encrypt_authorizers and contains_providers( nifi_config_dir + '/authorizers.xml', "authorizer"): encrypt_config_command += ('-a', nifi_config_dir + '/authorizers.xml') if last_config_version: last_config = get_config_by_version('/var/lib/ambari-agent/data', 'nifi-ambari-config', last_config_version) last_master_key_password = last_config['configurations'][ 'nifi-ambari-config'][ 'nifi.security.encrypt.configuration.password'] if last_master_key_password and last_master_key_password != master_key_password: encrypt_config_command += ( '-m', '-w', PasswordString(last_master_key_password)) encrypt_config_command += ('-p', PasswordString(master_key_password)) Execute(encrypt_config_command, user=nifi_user, logoutput=False, environment=environment) save_config_version(config_version_file, 'encrypt', current_version, nifi_user, nifi_group)