def setup_tls_toolkit_upgrade(self, env): import params env.set_params(params) if params.upgrade_direction == Direction.UPGRADE and params.nifi_ssl_enabled and params.nifi_ca_host: version_file = params.nifi_config_dir + '/config_version' client_json_file = params.nifi_config_dir + '/nifi-certificate-authority-client.json' if sudo.path_isfile(version_file): Logger.info(format('Remove config version file if does exist')) sudo.unlink(version_file) if sudo.path_isfile(client_json_file): Logger.info(format('Remove client json file')) sudo.unlink(client_json_file)
def stop(self, env, upgrade_type=None): import params import status_params nifi_toolkit_util_common.copy_toolkit_scripts( params.toolkit_files_dir, params.toolkit_tmp_dir, params.nifi_user, params.nifi_group, upgrade_type, service=nifi_toolkit_util_common.NIFI) run_ca_script = os.path.join(params.toolkit_tmp_dir, 'run_ca.sh') ca_server_script = nifi_toolkit_util_common.get_toolkit_script( 'tls-toolkit.sh', params.toolkit_tmp_dir, params.stack_version_buildnum) File(ca_server_script, mode=0755) File(run_ca_script, mode=0755) if path_isfile(status_params.nifi_ca_pid_file): Execute( (run_ca_script, "stop", params.jdk64_home, ca_server_script, params.nifi_config_dir + '/nifi-certificate-authority.json', params.nifi_ca_log_file_stdout, params.nifi_ca_log_file_stderr, status_params.nifi_ca_pid_file, params.toolkit_ca_check_url), user=params.nifi_user, logoutput=True) try: self.status(env) except ComponentIsNotRunning: unlink(status_params.nifi_ca_pid_file)
def osdisks(cls, config=None): """ Run df to find out the disks on the host. Only works on linux platforms. Note that this parser ignores any filesystems with spaces and any mounts with spaces. """ timeout = cls._get_mount_check_timeout(config) command = ["timeout", timeout, "df", "-kPT"] if not cls._check_remote_mounts(config): command.append("-l") df = subprocess.Popen(command, stdout=subprocess.PIPE) dfdata = df.communicate()[0] mounts = [ cls._parse_df_line(line) for line in dfdata.splitlines() if line ] result_mounts = [] for mount in mounts: if not mount: continue """ We need to filter mounts by several parameters: - mounted device is not in the ignored list - is accessible to user under which current process running - it is not file-mount (docker environment) """ if mount["device"] not in cls.IGNORE_DEVICES and \ mount["mountpoint"].split("/")[0] not in cls.IGNORE_ROOT_MOUNTS and\ cls._chk_writable_mount(mount['mountpoint']) and \ not path_isfile(mount["mountpoint"]): result_mounts.append(mount) return result_mounts
def setup_tls_toolkit_upgrade(self, env): import params env.set_params(params) version_file = params.nifi_config_dir + '/config_version' client_json_file = params.nifi_config_dir + '/nifi-certificate-authority-client.json' if not sudo.path_isfile(version_file): Logger.info(format('Create config version file if it does not exist')) version_num = params.config['configurationTags']['nifi-ambari-ssl-config']['tag'] nifi_toolkit_util.save_config_version(version_file, 'ssl', version_num, params.nifi_user, params.nifi_group) if sudo.path_isfile(client_json_file): Logger.info(format('Remove client json file')) sudo.unlink(client_json_file)
def get_toolkit_script(scriptName, scriptDir=files_dir, toolkitDirPrefix='nifi-toolkit-'): import params if params.stack_support_nifi_toolkit_package: result = os.path.join(params.stack_root, 'nifi-toolkit', 'bin', scriptName) Logger.info("Using repository toolkit script: " + result) return result else: Logger.info("Trying to find toolkit script dir with prefix: " + toolkitDirPrefix) prefixes = _get_best_prefixes_match(toolkitDirPrefix) for prefix in prefixes: for dir in os.listdir(scriptDir): if prefix in dir and dir.startswith('nifi-toolkit'): if prefix != toolkitDirPrefix: Logger.error( "Looks like your HDF stack repository not aligned to HDF mpack version, but we found " + dir + " toolkit directory, and will try to use it. If you facing some problems, please, " + "align your version as mentioned in documentation." ) nifiToolkitDir = os.path.join(scriptDir, dir) result = nifiToolkitDir + '/bin/' + scriptName if not sudo.path_isfile(result): raise Exception("Couldn't find file " + result) Logger.info("Using bundled toolkit script: " + result) return result raise Exception("Couldn't find nifi toolkit directory in " + scriptDir)
def get_config_version(version_file, version_type): if sudo.path_isfile(version_file): contents = sudo.read_file(version_file) version = json.loads(contents) if version_type in version: return version[version_type] else: return None
def move_store(client_dict, key): if store_exists(client_dict, key): num = 0 name = client_dict[key] while sudo.path_isfile(name + '.bak.' + str(num)): num += 1 sudo.copy(name, name + '.bak.' + str(num)) sudo.unlink(name)
def osdisks(self): """ Run df to find out the disks on the host. Only works on linux platforms. Note that this parser ignores any filesystems with spaces and any mounts with spaces. """ timeout = self._get_mount_check_timeout() command = ["timeout", timeout, "df", "-kPT"] blacklisted_mount_points = [] if self.config: ignore_mount_value = self.config.get("agent", "ignore_mount_points", default="") blacklisted_mount_points = [ item.strip() for item in ignore_mount_value.split(",") if len(item.strip()) != 0 ] if not self._check_remote_mounts(): command.append("-l") try: code, out, err = shell.call(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=int(timeout), quiet=True) dfdata = out except Exception as ex: logger.warn("Checking disk usage failed: " + str(ex)) dfdata = '' result_mounts = [] ignored_mounts = [] for mount in self._parse_df(dfdata.splitlines()): """ We need to filter mounts by several parameters: - mounted device is not in the ignored list - is accessible to user under which current process running - it is not file-mount (docker environment) - mount path or a part of mount path is not in the blacklist """ if mount["device"] not in self.IGNORE_DEVICES and\ mount["mountpoint"].split("/")[0] not in self.IGNORE_ROOT_MOUNTS and\ self._chk_writable_mount(mount['mountpoint']) and\ not path_isfile(mount["mountpoint"]) and\ not self._is_mount_blacklisted(blacklisted_mount_points, mount["mountpoint"]): result_mounts.append(mount) else: ignored_mounts.append(mount) if len(ignored_mounts) > 0: ignore_list = [el["mountpoint"] for el in ignored_mounts] logger.info("Some mount points were ignored: {0}".format( ', '.join(ignore_list))) return result_mounts
def action_create(self): filename = self.resource.filename comment_symbols = self.resource.comment_symbols delimiter = self.resource.key_value_delimiter properties = self.resource.properties unsaved_values = properties.keys() new_content_lines = [] final_content_lines = "" if sudo.path_isfile(filename): file_content = sudo.read_file(filename, encoding=self.resource.encoding) new_content_lines += file_content.split('\n') Logger.info( format("Modifying existing properties file: {filename}")) for line_num in range(len(new_content_lines)): line = new_content_lines[line_num] if line.lstrip() and not line.lstrip( )[0] in comment_symbols and delimiter in line: in_var_name = line.split(delimiter)[0].strip() in_var_value = line.split(delimiter)[1].strip() if in_var_name in properties: value = InlineTemplate(unicode( properties[in_var_name])).get_content() new_content_lines[line_num] = u"{0}{1}{2}".format( unicode(in_var_name), delimiter, value) unsaved_values.remove(in_var_name) else: Logger.info( format( "Creating new properties file as {filename} doesn't exist") ) for property_name in unsaved_values: value = InlineTemplate(unicode( properties[property_name])).get_content() line = u"{0}{1}{2}".format(unicode(property_name), delimiter, value) new_content_lines.append(line) final_content_lines = u"\n".join(new_content_lines) if not final_content_lines.endswith("\n"): final_content_lines = final_content_lines + "\n" File( filename, content=final_content_lines, owner=self.resource.owner, group=self.resource.group, mode=self.resource.mode, encoding=self.resource.encoding, )
def setup_tls_toolkit_upgrade(self,env): import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') if params.upgrade_direction == Direction.UPGRADE and params.nifi_ssl_enabled and params.nifi_ca_host: version_file = params.nifi_config_dir + '/config_version' client_json_file = params.nifi_config_dir+ '/nifi-certificate-authority-client.json' if sudo.path_isfile(version_file): Logger.info(format('Remove config version file if does exist')) sudo.unlink(version_file) if sudo.path_isfile(client_json_file): Logger.info(format('Remove client json file')) sudo.unlink(client_json_file)
def convert_properties_to_dict(prop_file): dict = {} if sudo.path_isfile(prop_file): lines = sudo.read_file(prop_file).split('\n') for line in lines: props = line.rstrip().split('=') if len(props) == 2: dict[props[0]] = props[1] elif len(props) == 1: dict[props[0]] = '' return dict
def get_toolkit_script(scriptName, scriptDir=files_dir): nifiToolkitDir = None for dir in os.listdir(scriptDir): if dir.startswith('nifi-toolkit-'): nifiToolkitDir = os.path.join(scriptDir, dir) if nifiToolkitDir is None: raise Exception("Couldn't find nifi toolkit directory in " + scriptDir) result = nifiToolkitDir + '/bin/' + scriptName if not sudo.path_isfile(result): raise Exception("Couldn't find file " + result) return result
def remove_config_version(version_file, version_type, nifi_user, nifi_group): if sudo.path_isfile(version_file): contents = sudo.read_file(version_file) version = json.loads(contents) version.pop(version_type, None) sudo.unlink(version_file) File(version_file, owner=nifi_user, group=nifi_group, mode=0600, content=json.dumps(version))
def copy_toolkit_scripts(toolkit_files_dir, toolkit_tmp_dir, user, group, upgrade_type, service): import params if service == NIFI: run_ca_tmp_script = os.path.join(toolkit_tmp_dir, 'run_ca.sh') new_run_ca_tmp_script = StaticFile("run_ca.sh") if not sudo.path_isfile(run_ca_tmp_script) or sudo.read_file( run_ca_tmp_script) != new_run_ca_tmp_script: File(format(run_ca_tmp_script), content=new_run_ca_tmp_script, mode=0755, owner=user, group=group) if not params.stack_support_nifi_toolkit_package: nifiToolkitDirFilesPath = None nifiToolkitDirTmpPath = None Logger.info("Toolkit files dir is " + toolkit_files_dir) Logger.info("Toolkit tmp dir is " + toolkit_tmp_dir) for dir in os.listdir(toolkit_files_dir): if dir.startswith('nifi-toolkit-'): nifiToolkitDirFilesPath = os.path.join(toolkit_files_dir, dir) nifiToolkitDirTmpPath = os.path.join(toolkit_tmp_dir, dir) if not sudo.path_isdir(nifiToolkitDirTmpPath) or not (upgrade_type is None): os.system("\cp -r " + nifiToolkitDirFilesPath + " " + toolkit_tmp_dir) Directory(nifiToolkitDirTmpPath, owner=user, group=group, create_parents=False, recursive_ownership=True, cd_access="a", mode=0755) os.system("\/var/lib/ambari-agent/ambari-sudo.sh chmod -R 755 " + nifiToolkitDirTmpPath) else: Logger.info("Changing owner of package files") package_toolkit_dir = os.path.join(params.stack_root, 'current', 'nifi-toolkit') Directory(package_toolkit_dir, owner=user, group=group, create_parents=False, recursive_ownership=True, cd_access="a", mode=0755, recursion_follow_links=True)
def osdisks(cls, config=None): """ Run df to find out the disks on the host. Only works on linux platforms. Note that this parser ignores any filesystems with spaces and any mounts with spaces. """ timeout = cls._get_mount_check_timeout(config) command = ["timeout", timeout, "df", "-kPT"] blacklisted_mount_points = [] if config: ignore_mount_value = config.get("agent", "ignore_mount_points", default="") blacklisted_mount_points = [ item.strip() for item in ignore_mount_value.split(",") ] if not cls._check_remote_mounts(config): command.append("-l") df = subprocess.Popen(command, stdout=subprocess.PIPE) dfdata = df.communicate()[0] mounts = [ cls._parse_df_line(line) for line in dfdata.splitlines() if line ] result_mounts = [] ignored_mounts = [] for mount in mounts: if not mount: continue """ We need to filter mounts by several parameters: - mounted device is not in the ignored list - is accessible to user under which current process running - it is not file-mount (docker environment) - mount path or a part of mount path is not in the blacklist """ if mount["device"] not in cls.IGNORE_DEVICES and\ mount["mountpoint"].split("/")[0] not in cls.IGNORE_ROOT_MOUNTS and\ cls._chk_writable_mount(mount['mountpoint']) and\ not path_isfile(mount["mountpoint"]) and\ not cls._is_mount_blacklisted(blacklisted_mount_points, mount["mountpoint"]): result_mounts.append(mount) else: ignored_mounts.append(mount) if len(ignored_mounts) > 0: ignore_list = [el["mountpoint"] for el in ignored_mounts] logger.info("Some mount points was ignored: {0}".format( ', '.join(ignore_list))) return result_mounts
def setup_keystore_truststore(is_starting, params, config_version_file): if is_starting: #check against last version to determine if key/trust has changed last_config_version = get_config_version(config_version_file, 'ssl') last_config = get_config_by_version('/var/lib/ambari-agent/data', 'nifi-registry-ambari-ssl-config', last_config_version) ca_client_dict = get_nifi_ca_client_dict(last_config, params) using_client_json = len(ca_client_dict) == 0 and sudo.path_isfile( params.nifi_registry_config_dir + '/nifi-certificate-authority-client.json') if using_client_json: ca_client_dict = load(params.nifi_config_dir + '/nifi-certificate-authority-client.json') changed_ks_ts = changed_keystore_truststore( ca_client_dict, params.nifi_ca_client_config, using_client_json) if not len(ca_client_dict) == 0 else True if params.nifi_toolkit_tls_regenerate: move_keystore_truststore(ca_client_dict) ca_client_dict = {} elif changed_ks_ts: move_keystore_truststore(ca_client_dict) if changed_ks_ts or params.nifi_toolkit_tls_regenerate: overlay(ca_client_dict, params.nifi_ca_client_config) updated_properties = run_toolkit_client( ca_client_dict, params.nifi_registry_config_dir, params.jdk64_home, params.nifi_toolkit_java_options, params.nifi_registry_user, params.nifi_registry_group, params.toolkit_tmp_dir, params.stack_support_toolkit_update) update_nifi_registry_properties(updated_properties, params.nifi_registry_properties) save_config_version(config_version_file, 'ssl', params.nifi_registry_ambari_ssl_config_version, params.nifi_registry_user, params.nifi_registry_group) elif using_client_json: save_config_version(config_version_file, 'ssl', params.nifi_registry_ambari_ssl_config_version, params.nifi_registry_user, params.nifi_registry_group) old_nifi_registry_properties = convert_properties_to_dict( params.nifi_registry_config_dir + '/nifi-registry.properties') return populate_ssl_properties(old_nifi_registry_properties, params.nifi_registry_properties, params) else: return params.nifi_registry_properties
def get_content(self): # absolute path if self.name.startswith(os.path.sep): path = self.name # relative path else: basedir = self.env.config.basedir path = os.path.join(basedir, "files", self.name) if not sudo.path_isfile(path) and not sudo.path_lexists(path): raise Fail("{0} Source file {1} is not found".format(repr(self), path)) return self.read_file(path)
def contains_providers(provider_file, tag): from xml.dom.minidom import parseString import xml.dom.minidom if sudo.path_isfile(provider_file): content = sudo.read_file(provider_file) dom = xml.dom.minidom.parseString(content) collection = dom.documentElement if collection.getElementsByTagName(tag): return True else: return False else: return False
def save_config_version(version_file, version_type, version_num, nifi_registry_user, nifi_registry_group): version = {} if sudo.path_isfile(version_file): contents = sudo.read_file(version_file) version = json.loads(contents) version[version_type] = version_num sudo.unlink(version_file) else: version[version_type] = version_num File(version_file, owner=nifi_registry_user, group=nifi_registry_group, mode=0600, content=json.dumps(version))
def stop(self, env): import status_params if path_isfile(status_params.nifi_ca_pid_file): try: self.status(env) pid = int(read_file(status_params.nifi_ca_pid_file)) for i in range(25): kill(pid, SIGTERM) time.sleep(1) self.status(env) kill(pid, SIGKILL) time.sleep(5) self.status(env) except ComponentIsNotRunning: unlink(status_params.nifi_ca_pid_file)
def convert_properties_to_dict(jdk64_home, nifi_registry_props_file, nifi_registry_bootstrap, toolkit_tmp_dir, stack_version_buildnum): dict = {} if sudo.path_isfile(nifi_registry_props_file): encrypt_tool_script = nifi_toolkit_util_common.get_toolkit_script('encrypt-config.sh', toolkit_tmp_dir, stack_version_buildnum) File(encrypt_tool_script, mode=0755) command = 'ambari-sudo.sh JAVA_HOME=' + jdk64_home + ' '+ encrypt_tool_script + ' --nifiRegistry --decrypt -r ' + nifi_registry_props_file + ' -b ' + nifi_registry_bootstrap code, out = shell.call(command,quiet=True,logoutput=False) lines = out.split('\n') for line in lines: props = line.rstrip().split('=') if len(props) == 2: dict[props[0]] = props[1] elif len(props) == 1: dict[props[0]] = '' return dict
def configure(self, env, isInstall=False, is_starting = False): import params import status_params env.set_params(params) env.set_params(status_params) #create the log, pid, conf dirs if not already present nifi_dirs = [status_params.nifi_pid_dir, params.nifi_internal_dir, params.nifi_database_dir, params.nifi_flowfile_repo_dir, params.nifi_provenance_repo_dir_default, params.nifi_config_dir, params.nifi_flow_config_dir, params.nifi_state_dir, params.lib_dir] nifi_dirs.extend(params.nifi_content_repo_dirs) Directory(nifi_dirs, owner=params.nifi_user, group=params.nifi_group, create_parents=True, recursive_ownership=True, cd_access='a') # On some OS this folder may not exist, so we will create it before pushing files there Directory(params.limits_conf_dir, create_parents = True, owner='root', group='root' ) File(os.path.join(params.limits_conf_dir, 'nifi.conf'), owner='root', group='root', mode=0644, content=Template("nifi.conf.j2") ) #determine whether or not a cluster already exists based on zookeeper entries and determine if this is the first start of this node #if so authorizations and flow file will not be writen if not sudo.path_isfile(params.nifi_flow_config_dir+'/flow.xml.gz') and nifi_toolkit_util_common.existing_cluster(params): params.is_additional_node = True #write configurations self.write_configurations(params, is_starting) generate_logfeeder_input_config('nifi', Template("input.config-nifi.json.j2", extra_imports=[default])) # if this is not an additional node being added to an existing cluster write out flow.xml.gz to internal dir only if AMS installed (must be writable by Nifi) # and only during first install. It is used to automate setup of Ambari metrics reporting task in Nifi if not params.is_additional_node: if params.metrics_collector_host and params.nifi_ambari_reporting_enabled and not sudo.path_isfile(params.nifi_flow_config_dir+'/flow.xml.gz'): Execute('echo "First time setup so generating flow.xml.gz" >> ' + params.nifi_node_log_file, user=params.nifi_user) flow_content=InlineTemplate(params.nifi_flow_content) File(format("{params.nifi_flow_config_dir}/flow.xml"), content=flow_content, owner=params.nifi_user, group=params.nifi_group, mode=0600) Execute(format("cd {params.nifi_flow_config_dir}; gzip flow.xml;"), user=params.nifi_user)
def setup_keystore_truststore(self, is_starting, params, config_version_file): if is_starting: # check against last version to determine if key/trust has changed last_config_version = nifi_toolkit_util.get_config_version(config_version_file, 'ssl') last_config = nifi_toolkit_util.get_config_by_version('/var/lib/ambari-agent/data', 'nifi-ambari-ssl-config', last_config_version) ca_client_dict = nifi_toolkit_util.get_nifi_ca_client_dict(last_config, params) using_client_json = len(ca_client_dict) == 0 and sudo.path_isfile( params.nifi_config_dir + '/nifi-certificate-authority-client.json') if using_client_json: ca_client_dict = nifi_toolkit_util.load( params.nifi_config_dir + '/nifi-certificate-authority-client.json') changed_keystore_truststore = nifi_toolkit_util.changed_keystore_truststore(ca_client_dict, params.nifi_ca_client_config, using_client_json) if not len( ca_client_dict) == 0 else True if params.nifi_toolkit_tls_regenerate: nifi_toolkit_util.move_keystore_truststore(ca_client_dict) ca_client_dict = {} elif changed_keystore_truststore: nifi_toolkit_util.move_keystore_truststore(ca_client_dict) if changed_keystore_truststore or params.nifi_toolkit_tls_regenerate: nifi_toolkit_util.overlay(ca_client_dict, params.nifi_ca_client_config) updated_properties = self.run_toolkit_client(ca_client_dict, params.nifi_config_dir, params.jdk64_home, params.nifi_user, params.nifi_group, params.stack_support_toolkit_update) nifi_toolkit_util.update_nifi_properties(updated_properties, params.nifi_properties) nifi_toolkit_util.save_config_version(config_version_file, 'ssl', params.nifi_ambari_ssl_config_version, params.nifi_user, params.nifi_group) elif using_client_json: nifi_toolkit_util.save_config_version(config_version_file, 'ssl', params.nifi_ambari_ssl_config_version, params.nifi_user, params.nifi_group) old_nifi_properties = nifi_toolkit_util.convert_properties_to_dict( params.nifi_config_dir + '/nifi.properties') return nifi_toolkit_util.populate_ssl_properties(old_nifi_properties, params.nifi_properties, params) else: return params.nifi_properties
def setup_tls_toolkit_upgrade(self,env): import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') if params.upgrade_direction == Direction.UPGRADE and params.nifi_registry_ssl_enabled and params.nifi_ca_host: version_file = params.nifi_registry_config_dir + '/config_version' client_json_file = params.nifi_registry_config_dir+ '/nifi-certificate-authority-client.json' # if not sudo.path_isfile(version_file): # Logger.info(format('Create config version file if it does not exist')) # version_num = params.config['configurationTags']['nifi-registry-ambari-ssl-config']['tag'] # nifi_toolkit_util.save_config_version(version_file,'ssl',version_num,params.nifi_registry_user,params.nifi_registry_group) if sudo.path_isfile(client_json_file): Logger.info(format('Remove client json file')) sudo.unlink(client_json_file)
def action_create(self): filename = self.resource.filename comment_symbols = self.resource.comment_symbols delimiter = self.resource.key_value_delimiter properties = self.resource.properties unsaved_values = properties.keys() new_content_lines = [] if sudo.path_isfile(filename): file_content = sudo.read_file(filename, encoding=self.resource.encoding) new_content_lines += file_content.split('\n') Logger.info(format("Modifying existing properties file: {filename}")) for line_num in range(len(new_content_lines)): line = new_content_lines[line_num] if line.lstrip() and not line.lstrip()[0] in comment_symbols and delimiter in line: in_var_name = line.split(delimiter)[0].strip() in_var_value = line.split(delimiter)[1].strip() if in_var_name in properties: value = InlineTemplate(unicode(properties[in_var_name])).get_content() new_content_lines[line_num] = u"{0}{1}{2}".format(unicode(in_var_name), delimiter, value) unsaved_values.remove(in_var_name) else: Logger.info(format("Creating new properties file as {filename} doesn't exist")) for property_name in unsaved_values: value = InlineTemplate(unicode(properties[property_name])).get_content() line = u"{0}{1}{2}".format(unicode(property_name), delimiter, value) new_content_lines.append(line) with Environment.get_instance_copy() as env: File (filename, content = u"\n".join(new_content_lines) + "\n", owner = self.resource.owner, group = self.resource.group, mode = self.resource.mode, encoding = self.resource.encoding, )
def encrypt_sensitive_properties(self, config_version_file, current_version, nifi_config_dir, jdk64_home, nifi_user, nifi_group, master_key_password, nifi_flow_config_dir, nifi_sensitive_props_key, is_starting): Logger.info("Encrypting NiFi sensitive configuration properties") encrypt_config_script = nifi_toolkit_util.get_toolkit_script('encrypt-config.sh') encrypt_config_script_prefix = ('JAVA_HOME=' + jdk64_home, encrypt_config_script) File(encrypt_config_script, mode=0755) if is_starting: last_master_key_password = None last_config_version = nifi_toolkit_util.get_config_version(config_version_file, 'encrypt') encrypt_config_script_params = ('-v', '-b', nifi_config_dir + '/bootstrap.conf') encrypt_config_script_params = encrypt_config_script_params + ('-n', nifi_config_dir + '/nifi.properties') if sudo.path_isfile(nifi_flow_config_dir + '/flow.xml.gz') and len( sudo.read_file(nifi_flow_config_dir + '/flow.xml.gz')) > 0: encrypt_config_script_params = encrypt_config_script_params + ( '-f', nifi_flow_config_dir + '/flow.xml.gz', '-s', PasswordString(nifi_sensitive_props_key)) if nifi_toolkit_util.contains_providers(nifi_config_dir + '/login-identity-providers.xml'): encrypt_config_script_params = encrypt_config_script_params + ( '-l', nifi_config_dir + '/login-identity-providers.xml') if last_config_version: last_config = nifi_toolkit_util.get_config_by_version('/var/lib/ambari-agent/data', 'nifi-ambari-config', last_config_version) last_master_key_password = last_config['configurations']['nifi-ambari-config'][ 'nifi.security.encrypt.configuration.password'] if last_master_key_password and last_master_key_password != master_key_password: encrypt_config_script_params = encrypt_config_script_params + ( '-m', '-w', PasswordString(last_master_key_password)) encrypt_config_script_params = encrypt_config_script_params + ('-p', PasswordString(master_key_password)) encrypt_config_script_prefix = encrypt_config_script_prefix + encrypt_config_script_params Execute(encrypt_config_script_prefix, user=nifi_user, logoutput=False) nifi_toolkit_util.save_config_version(config_version_file, 'encrypt', current_version, nifi_user, nifi_group)
def get_toolkit_script(scriptName, scriptDir=files_dir, toolkitDirPrefix='nifi-toolkit-'): import params if params.stack_support_nifi_toolkit_package: result = os.path.join(params.stack_root, 'current', 'nifi-toolkit', 'bin', scriptName) Logger.info("Using repository toolkit script: " + result) return result else: Logger.info("Trying to find toolkit script dir with prefix: " + toolkitDirPrefix) # in case user did not aligned mpack and repo version we can fallback to partly-match toolkit version # for example DDP=3.2.1.0-111, and toolkit bundled is 3.2.0.0-333, code will check for prefixes # 3.2.1.0, 3.2.1, 3.2, and as result will use 3.2.0.0-333 toolkit, most likely this will work prefixes = _get_best_prefixes_match(toolkitDirPrefix) for prefix in prefixes: for dir in os.listdir(scriptDir): if prefix in dir and dir.startswith('nifi-toolkit'): if prefix != toolkitDirPrefix: Logger.error( "Looks like your DDP stack repository not aligned to DDP mpack version, but we found " + dir + " toolkit directory, and will try to use it. If you facing some problems, please, " + "align your version as mentioned in documentation." ) nifiToolkitDir = os.path.join(scriptDir, dir) result = nifiToolkitDir + '/bin/' + scriptName if not sudo.path_isfile(result): raise Exception("Couldn't find file " + result) Logger.info("Using bundled toolkit script: " + result) return result raise Exception("Couldn't find nifi toolkit directory in " + scriptDir)
def encrypt_sensitive_properties( nifi_config_dir, jdk64_home, java_options, nifi_user, last_master_key, master_key_password, is_starting, toolkit_tmp_dir, stack_version_buildnum, service, nifi_flow_config_dir=None, nifi_sensitive_props_key=None, support_encrypt_authorizers=None, ): encrypt_config_script = get_toolkit_script('encrypt-config.sh', toolkit_tmp_dir, stack_version_buildnum) encrypt_config_command = (encrypt_config_script, ) environment = {'JAVA_HOME': jdk64_home, 'JAVA_OPTS': java_options} File(encrypt_config_script, mode=0755) if is_starting: if service == NIFI: Logger.info("Encrypting NiFi sensitive configuration properties") encrypt_config_command += ('-v', '-b', nifi_config_dir + '/bootstrap.conf') encrypt_config_command += ('-n', nifi_config_dir + '/nifi.properties') if (sudo.path_isfile(nifi_flow_config_dir + '/flow.xml.gz') and len(sudo.read_file(nifi_flow_config_dir + '/flow.xml.gz')) > 0): encrypt_config_command += ( '-f', nifi_flow_config_dir + '/flow.xml.gz', '-s', PasswordString(nifi_sensitive_props_key)) if contains_providers( nifi_config_dir + '/login-identity-providers.xml', "provider"): encrypt_config_command += ('-l', nifi_config_dir + '/login-identity-providers.xml') if support_encrypt_authorizers and contains_providers( nifi_config_dir + '/authorizers.xml', "authorizer"): encrypt_config_command += ('-a', nifi_config_dir + '/authorizers.xml') if last_master_key: encrypt_config_command += ('-m', '-e', PasswordString(last_master_key)) encrypt_config_command += ('-p', PasswordString(master_key_password)) Execute(encrypt_config_command, user=nifi_user, logoutput=False, environment=environment) elif service == NIFI_REGISTRY: Logger.info( "Encrypting NiFi Registry sensitive configuration properties") encrypt_config_command += ('--nifiRegistry', '-v', '-b', nifi_config_dir + '/bootstrap.conf') encrypt_config_command += ('-r', nifi_config_dir + '/nifi-registry.properties') if contains_providers(nifi_config_dir + '/identity-providers.xml', "provider"): encrypt_config_command += ('-i', nifi_config_dir + '/identity-providers.xml') if contains_providers(nifi_config_dir + '/authorizers.xml', "authorizer"): encrypt_config_command += ('-a', nifi_config_dir + '/authorizers.xml') if last_master_key: encrypt_config_command += ('--oldKey', PasswordString(last_master_key)) encrypt_config_command += ('-p', PasswordString(master_key_password)) Execute(encrypt_config_command, user=nifi_user, logoutput=False, environment=environment)
'nifi.node.port'] java_home = config['ambariLevelParams']['java_home'] security_enabled = config['configurations']['cluster-env']['security_enabled'] smokeuser = config['configurations']['cluster-env']['smokeuser'] smokeuser_principal = config['configurations']['cluster-env'][ 'smokeuser_principal_name'] smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] kinit_path_local = get_kinit_path( default('/configurations/kerberos-env/executable_search_paths', None)) stack_support_nifi_toolkit_package = check_stack_feature( 'nifi_toolkit_package', version_for_stack_feature_checks) #some released HDP stacks will not have this stack feature, manually check if not stack_support_nifi_toolkit_package and stack_name == "HDP": marker_script = os.path.join(stack_root, "current/nifi-toolkit/bin/tls-toolkit.sh") if sudo.path_isfile(marker_script): stack_support_nifi_toolkit_package = True stack_support_encrypt_config = check_stack_feature( 'nifi_encrypt_config', version_for_stack_feature_checks) stack_support_toolkit_update = check_stack_feature( 'toolkit_config_update', version_for_stack_feature_checks) stack_support_admin_toolkit = check_stack_feature( 'admin_toolkit_support', version_for_stack_feature_checks) stack_support_nifi_jaas = check_stack_feature( 'nifi_jaas_conf_create', version_for_stack_feature_checks) stack_support_encrypt_authorizers = check_stack_feature( 'nifi_encrypted_authorizers_config', version_for_stack_feature_checks) stack_support_nifi_auto_client_registration = check_stack_feature( 'nifi_auto_client_registration', version_for_stack_feature_checks) if security_enabled:
def store_exists(client_dict, key): if key not in client_dict: return False return sudo.path_isfile(client_dict[key])
def load(config_json): if sudo.path_isfile(config_json): contents = sudo.read_file(config_json) if len(contents) > 0: return json.loads(contents) return {}