def setup_tls_toolkit_upgrade(self, env):
import params
env.set_params(params)
if params.upgrade_direction == Direction.UPGRADE and params.nifi_ssl_enabled and params.nifi_ca_host:
version_file = params.nifi_config_dir + '/config_version'
client_json_file = params.nifi_config_dir + '/nifi-certificate-authority-client.json'
if sudo.path_isfile(version_file):
Logger.info(format('Remove config version file if does exist'))
sudo.unlink(version_file)
if sudo.path_isfile(client_json_file):
Logger.info(format('Remove client json file'))
sudo.unlink(client_json_file)
def stop(self, env, upgrade_type=None):
import params
import status_params
nifi_toolkit_util_common.copy_toolkit_scripts(
params.toolkit_files_dir,
params.toolkit_tmp_dir,
params.nifi_user,
params.nifi_group,
upgrade_type,
service=nifi_toolkit_util_common.NIFI)
run_ca_script = os.path.join(params.toolkit_tmp_dir, 'run_ca.sh')
ca_server_script = nifi_toolkit_util_common.get_toolkit_script(
'tls-toolkit.sh', params.toolkit_tmp_dir,
params.stack_version_buildnum)
File(ca_server_script, mode=0755)
File(run_ca_script, mode=0755)
if path_isfile(status_params.nifi_ca_pid_file):
Execute(
(run_ca_script, "stop", params.jdk64_home, ca_server_script,
params.nifi_config_dir + '/nifi-certificate-authority.json',
params.nifi_ca_log_file_stdout,
params.nifi_ca_log_file_stderr,
status_params.nifi_ca_pid_file, params.toolkit_ca_check_url),
user=params.nifi_user,
logoutput=True)
try:
self.status(env)
except ComponentIsNotRunning:
unlink(status_params.nifi_ca_pid_file)
def osdisks(cls, config=None):
""" Run df to find out the disks on the host. Only works on linux
platforms. Note that this parser ignores any filesystems with spaces
and any mounts with spaces. """
timeout = cls._get_mount_check_timeout(config)
command = ["timeout", timeout, "df", "-kPT"]
if not cls._check_remote_mounts(config):
command.append("-l")
df = subprocess.Popen(command, stdout=subprocess.PIPE)
dfdata = df.communicate()[0]
mounts = [
cls._parse_df_line(line) for line in dfdata.splitlines() if line
]
result_mounts = []
for mount in mounts:
if not mount:
continue
"""
We need to filter mounts by several parameters:
- mounted device is not in the ignored list
- is accessible to user under which current process running
- it is not file-mount (docker environment)
"""
if mount["device"] not in cls.IGNORE_DEVICES and \
mount["mountpoint"].split("/")[0] not in cls.IGNORE_ROOT_MOUNTS and\
cls._chk_writable_mount(mount['mountpoint']) and \
not path_isfile(mount["mountpoint"]):
result_mounts.append(mount)
return result_mounts
def setup_tls_toolkit_upgrade(self, env):
import params
env.set_params(params)
version_file = params.nifi_config_dir + '/config_version'
client_json_file = params.nifi_config_dir + '/nifi-certificate-authority-client.json'
if not sudo.path_isfile(version_file):
Logger.info(format('Create config version file if it does not exist'))
version_num = params.config['configurationTags']['nifi-ambari-ssl-config']['tag']
nifi_toolkit_util.save_config_version(version_file, 'ssl', version_num, params.nifi_user,
params.nifi_group)
if sudo.path_isfile(client_json_file):
Logger.info(format('Remove client json file'))
sudo.unlink(client_json_file)
def get_toolkit_script(scriptName,
scriptDir=files_dir,
toolkitDirPrefix='nifi-toolkit-'):
import params
if params.stack_support_nifi_toolkit_package:
result = os.path.join(params.stack_root, 'nifi-toolkit', 'bin',
scriptName)
Logger.info("Using repository toolkit script: " + result)
return result
else:
Logger.info("Trying to find toolkit script dir with prefix: " +
toolkitDirPrefix)
prefixes = _get_best_prefixes_match(toolkitDirPrefix)
for prefix in prefixes:
for dir in os.listdir(scriptDir):
if prefix in dir and dir.startswith('nifi-toolkit'):
if prefix != toolkitDirPrefix:
Logger.error(
"Looks like your HDF stack repository not aligned to HDF mpack version, but we found "
+ dir +
" toolkit directory, and will try to use it. If you facing some problems, please, "
+
"align your version as mentioned in documentation."
)
nifiToolkitDir = os.path.join(scriptDir, dir)
result = nifiToolkitDir + '/bin/' + scriptName
if not sudo.path_isfile(result):
raise Exception("Couldn't find file " + result)
Logger.info("Using bundled toolkit script: " + result)
return result
raise Exception("Couldn't find nifi toolkit directory in " + scriptDir)
def get_config_version(version_file, version_type):
if sudo.path_isfile(version_file):
contents = sudo.read_file(version_file)
version = json.loads(contents)
if version_type in version:
return version[version_type]
else:
return None
def move_store(client_dict, key):
if store_exists(client_dict, key):
num = 0
name = client_dict[key]
while sudo.path_isfile(name + '.bak.' + str(num)):
num += 1
sudo.copy(name, name + '.bak.' + str(num))
sudo.unlink(name)
def osdisks(self):
""" Run df to find out the disks on the host. Only works on linux
platforms. Note that this parser ignores any filesystems with spaces
and any mounts with spaces. """
timeout = self._get_mount_check_timeout()
command = ["timeout", timeout, "df", "-kPT"]
blacklisted_mount_points = []
if self.config:
ignore_mount_value = self.config.get("agent",
"ignore_mount_points",
default="")
blacklisted_mount_points = [
item.strip() for item in ignore_mount_value.split(",")
if len(item.strip()) != 0
]
if not self._check_remote_mounts():
command.append("-l")
try:
code, out, err = shell.call(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=int(timeout),
quiet=True)
dfdata = out
except Exception as ex:
logger.warn("Checking disk usage failed: " + str(ex))
dfdata = ''
result_mounts = []
ignored_mounts = []
for mount in self._parse_df(dfdata.splitlines()):
"""
We need to filter mounts by several parameters:
- mounted device is not in the ignored list
- is accessible to user under which current process running
- it is not file-mount (docker environment)
- mount path or a part of mount path is not in the blacklist
"""
if mount["device"] not in self.IGNORE_DEVICES and\
mount["mountpoint"].split("/")[0] not in self.IGNORE_ROOT_MOUNTS and\
self._chk_writable_mount(mount['mountpoint']) and\
not path_isfile(mount["mountpoint"]) and\
not self._is_mount_blacklisted(blacklisted_mount_points, mount["mountpoint"]):
result_mounts.append(mount)
else:
ignored_mounts.append(mount)
if len(ignored_mounts) > 0:
ignore_list = [el["mountpoint"] for el in ignored_mounts]
logger.info("Some mount points were ignored: {0}".format(
', '.join(ignore_list)))
return result_mounts
def action_create(self):
filename = self.resource.filename
comment_symbols = self.resource.comment_symbols
delimiter = self.resource.key_value_delimiter
properties = self.resource.properties
unsaved_values = properties.keys()
new_content_lines = []
final_content_lines = ""
if sudo.path_isfile(filename):
file_content = sudo.read_file(filename,
encoding=self.resource.encoding)
new_content_lines += file_content.split('\n')
Logger.info(
format("Modifying existing properties file: {filename}"))
for line_num in range(len(new_content_lines)):
line = new_content_lines[line_num]
if line.lstrip() and not line.lstrip(
)[0] in comment_symbols and delimiter in line:
in_var_name = line.split(delimiter)[0].strip()
in_var_value = line.split(delimiter)[1].strip()
if in_var_name in properties:
value = InlineTemplate(unicode(
properties[in_var_name])).get_content()
new_content_lines[line_num] = u"{0}{1}{2}".format(
unicode(in_var_name), delimiter, value)
unsaved_values.remove(in_var_name)
else:
Logger.info(
format(
"Creating new properties file as {filename} doesn't exist")
)
for property_name in unsaved_values:
value = InlineTemplate(unicode(
properties[property_name])).get_content()
line = u"{0}{1}{2}".format(unicode(property_name), delimiter,
value)
new_content_lines.append(line)
final_content_lines = u"\n".join(new_content_lines)
if not final_content_lines.endswith("\n"):
final_content_lines = final_content_lines + "\n"
File(
filename,
content=final_content_lines,
owner=self.resource.owner,
group=self.resource.group,
mode=self.resource.mode,
encoding=self.resource.encoding,
)
def setup_tls_toolkit_upgrade(self,env):
import params
env.set_params(params)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None:
raise Fail('Unable to determine the stack and stack version')
if params.upgrade_direction == Direction.UPGRADE and params.nifi_ssl_enabled and params.nifi_ca_host:
version_file = params.nifi_config_dir + '/config_version'
client_json_file = params.nifi_config_dir+ '/nifi-certificate-authority-client.json'
if sudo.path_isfile(version_file):
Logger.info(format('Remove config version file if does exist'))
sudo.unlink(version_file)
if sudo.path_isfile(client_json_file):
Logger.info(format('Remove client json file'))
sudo.unlink(client_json_file)
def convert_properties_to_dict(prop_file):
dict = {}
if sudo.path_isfile(prop_file):
lines = sudo.read_file(prop_file).split('\n')
for line in lines:
props = line.rstrip().split('=')
if len(props) == 2:
dict[props[0]] = props[1]
elif len(props) == 1:
dict[props[0]] = ''
return dict
def get_toolkit_script(scriptName, scriptDir=files_dir):
nifiToolkitDir = None
for dir in os.listdir(scriptDir):
if dir.startswith('nifi-toolkit-'):
nifiToolkitDir = os.path.join(scriptDir, dir)
if nifiToolkitDir is None:
raise Exception("Couldn't find nifi toolkit directory in " + scriptDir)
result = nifiToolkitDir + '/bin/' + scriptName
if not sudo.path_isfile(result):
raise Exception("Couldn't find file " + result)
return result
def remove_config_version(version_file, version_type, nifi_user, nifi_group):
if sudo.path_isfile(version_file):
contents = sudo.read_file(version_file)
version = json.loads(contents)
version.pop(version_type, None)
sudo.unlink(version_file)
File(version_file,
owner=nifi_user,
group=nifi_group,
mode=0600,
content=json.dumps(version))
def copy_toolkit_scripts(toolkit_files_dir, toolkit_tmp_dir, user, group,
upgrade_type, service):
import params
if service == NIFI:
run_ca_tmp_script = os.path.join(toolkit_tmp_dir, 'run_ca.sh')
new_run_ca_tmp_script = StaticFile("run_ca.sh")
if not sudo.path_isfile(run_ca_tmp_script) or sudo.read_file(
run_ca_tmp_script) != new_run_ca_tmp_script:
File(format(run_ca_tmp_script),
content=new_run_ca_tmp_script,
mode=0755,
owner=user,
group=group)
if not params.stack_support_nifi_toolkit_package:
nifiToolkitDirFilesPath = None
nifiToolkitDirTmpPath = None
Logger.info("Toolkit files dir is " + toolkit_files_dir)
Logger.info("Toolkit tmp dir is " + toolkit_tmp_dir)
for dir in os.listdir(toolkit_files_dir):
if dir.startswith('nifi-toolkit-'):
nifiToolkitDirFilesPath = os.path.join(toolkit_files_dir, dir)
nifiToolkitDirTmpPath = os.path.join(toolkit_tmp_dir, dir)
if not sudo.path_isdir(nifiToolkitDirTmpPath) or not (upgrade_type is
None):
os.system("\cp -r " + nifiToolkitDirFilesPath + " " +
toolkit_tmp_dir)
Directory(nifiToolkitDirTmpPath,
owner=user,
group=group,
create_parents=False,
recursive_ownership=True,
cd_access="a",
mode=0755)
os.system("\/var/lib/ambari-agent/ambari-sudo.sh chmod -R 755 " +
nifiToolkitDirTmpPath)
else:
Logger.info("Changing owner of package files")
package_toolkit_dir = os.path.join(params.stack_root, 'current',
'nifi-toolkit')
Directory(package_toolkit_dir,
owner=user,
group=group,
create_parents=False,
recursive_ownership=True,
cd_access="a",
mode=0755,
recursion_follow_links=True)
def osdisks(cls, config=None):
""" Run df to find out the disks on the host. Only works on linux
platforms. Note that this parser ignores any filesystems with spaces
and any mounts with spaces. """
timeout = cls._get_mount_check_timeout(config)
command = ["timeout", timeout, "df", "-kPT"]
blacklisted_mount_points = []
if config:
ignore_mount_value = config.get("agent",
"ignore_mount_points",
default="")
blacklisted_mount_points = [
item.strip() for item in ignore_mount_value.split(",")
]
if not cls._check_remote_mounts(config):
command.append("-l")
df = subprocess.Popen(command, stdout=subprocess.PIPE)
dfdata = df.communicate()[0]
mounts = [
cls._parse_df_line(line) for line in dfdata.splitlines() if line
]
result_mounts = []
ignored_mounts = []
for mount in mounts:
if not mount:
continue
"""
We need to filter mounts by several parameters:
- mounted device is not in the ignored list
- is accessible to user under which current process running
- it is not file-mount (docker environment)
- mount path or a part of mount path is not in the blacklist
"""
if mount["device"] not in cls.IGNORE_DEVICES and\
mount["mountpoint"].split("/")[0] not in cls.IGNORE_ROOT_MOUNTS and\
cls._chk_writable_mount(mount['mountpoint']) and\
not path_isfile(mount["mountpoint"]) and\
not cls._is_mount_blacklisted(blacklisted_mount_points, mount["mountpoint"]):
result_mounts.append(mount)
else:
ignored_mounts.append(mount)
if len(ignored_mounts) > 0:
ignore_list = [el["mountpoint"] for el in ignored_mounts]
logger.info("Some mount points was ignored: {0}".format(
', '.join(ignore_list)))
return result_mounts
def setup_keystore_truststore(is_starting, params, config_version_file):
if is_starting:
#check against last version to determine if key/trust has changed
last_config_version = get_config_version(config_version_file, 'ssl')
last_config = get_config_by_version('/var/lib/ambari-agent/data',
'nifi-registry-ambari-ssl-config',
last_config_version)
ca_client_dict = get_nifi_ca_client_dict(last_config, params)
using_client_json = len(ca_client_dict) == 0 and sudo.path_isfile(
params.nifi_registry_config_dir +
'/nifi-certificate-authority-client.json')
if using_client_json:
ca_client_dict = load(params.nifi_config_dir +
'/nifi-certificate-authority-client.json')
changed_ks_ts = changed_keystore_truststore(
ca_client_dict, params.nifi_ca_client_config,
using_client_json) if not len(ca_client_dict) == 0 else True
if params.nifi_toolkit_tls_regenerate:
move_keystore_truststore(ca_client_dict)
ca_client_dict = {}
elif changed_ks_ts:
move_keystore_truststore(ca_client_dict)
if changed_ks_ts or params.nifi_toolkit_tls_regenerate:
overlay(ca_client_dict, params.nifi_ca_client_config)
updated_properties = run_toolkit_client(
ca_client_dict, params.nifi_registry_config_dir,
params.jdk64_home, params.nifi_toolkit_java_options,
params.nifi_registry_user, params.nifi_registry_group,
params.toolkit_tmp_dir, params.stack_support_toolkit_update)
update_nifi_registry_properties(updated_properties,
params.nifi_registry_properties)
save_config_version(config_version_file, 'ssl',
params.nifi_registry_ambari_ssl_config_version,
params.nifi_registry_user,
params.nifi_registry_group)
elif using_client_json:
save_config_version(config_version_file, 'ssl',
params.nifi_registry_ambari_ssl_config_version,
params.nifi_registry_user,
params.nifi_registry_group)
old_nifi_registry_properties = convert_properties_to_dict(
params.nifi_registry_config_dir + '/nifi-registry.properties')
return populate_ssl_properties(old_nifi_registry_properties,
params.nifi_registry_properties, params)
else:
return params.nifi_registry_properties
def get_content(self):
# absolute path
if self.name.startswith(os.path.sep):
path = self.name
# relative path
else:
basedir = self.env.config.basedir
path = os.path.join(basedir, "files", self.name)
if not sudo.path_isfile(path) and not sudo.path_lexists(path):
raise Fail("{0} Source file {1} is not found".format(repr(self), path))
return self.read_file(path)
def contains_providers(provider_file, tag):
from xml.dom.minidom import parseString
import xml.dom.minidom
if sudo.path_isfile(provider_file):
content = sudo.read_file(provider_file)
dom = xml.dom.minidom.parseString(content)
collection = dom.documentElement
if collection.getElementsByTagName(tag):
return True
else:
return False
else:
return False
def save_config_version(version_file, version_type, version_num,
nifi_registry_user, nifi_registry_group):
version = {}
if sudo.path_isfile(version_file):
contents = sudo.read_file(version_file)
version = json.loads(contents)
version[version_type] = version_num
sudo.unlink(version_file)
else:
version[version_type] = version_num
File(version_file,
owner=nifi_registry_user,
group=nifi_registry_group,
mode=0600,
content=json.dumps(version))
def stop(self, env):
import status_params
if path_isfile(status_params.nifi_ca_pid_file):
try:
self.status(env)
pid = int(read_file(status_params.nifi_ca_pid_file))
for i in range(25):
kill(pid, SIGTERM)
time.sleep(1)
self.status(env)
kill(pid, SIGKILL)
time.sleep(5)
self.status(env)
except ComponentIsNotRunning:
unlink(status_params.nifi_ca_pid_file)
def convert_properties_to_dict(jdk64_home, nifi_registry_props_file, nifi_registry_bootstrap, toolkit_tmp_dir, stack_version_buildnum):
dict = {}
if sudo.path_isfile(nifi_registry_props_file):
encrypt_tool_script = nifi_toolkit_util_common.get_toolkit_script('encrypt-config.sh', toolkit_tmp_dir, stack_version_buildnum)
File(encrypt_tool_script, mode=0755)
command = 'ambari-sudo.sh JAVA_HOME=' + jdk64_home + ' '+ encrypt_tool_script + ' --nifiRegistry --decrypt -r ' + nifi_registry_props_file + ' -b ' + nifi_registry_bootstrap
code, out = shell.call(command,quiet=True,logoutput=False)
lines = out.split('\n')
for line in lines:
props = line.rstrip().split('=')
if len(props) == 2:
dict[props[0]] = props[1]
elif len(props) == 1:
dict[props[0]] = ''
return dict
def configure(self, env, isInstall=False, is_starting = False):
import params
import status_params
env.set_params(params)
env.set_params(status_params)
#create the log, pid, conf dirs if not already present
nifi_dirs = [status_params.nifi_pid_dir, params.nifi_internal_dir, params.nifi_database_dir, params.nifi_flowfile_repo_dir, params.nifi_provenance_repo_dir_default, params.nifi_config_dir, params.nifi_flow_config_dir, params.nifi_state_dir, params.lib_dir]
nifi_dirs.extend(params.nifi_content_repo_dirs)
Directory(nifi_dirs, owner=params.nifi_user, group=params.nifi_group, create_parents=True, recursive_ownership=True, cd_access='a')
# On some OS this folder may not exist, so we will create it before pushing files there
Directory(params.limits_conf_dir,
create_parents = True,
owner='root',
group='root'
)
File(os.path.join(params.limits_conf_dir, 'nifi.conf'),
owner='root',
group='root',
mode=0644,
content=Template("nifi.conf.j2")
)
#determine whether or not a cluster already exists based on zookeeper entries and determine if this is the first start of this node
#if so authorizations and flow file will not be writen
if not sudo.path_isfile(params.nifi_flow_config_dir+'/flow.xml.gz') and nifi_toolkit_util_common.existing_cluster(params):
params.is_additional_node = True
#write configurations
self.write_configurations(params, is_starting)
generate_logfeeder_input_config('nifi', Template("input.config-nifi.json.j2", extra_imports=[default]))
# if this is not an additional node being added to an existing cluster write out flow.xml.gz to internal dir only if AMS installed (must be writable by Nifi)
# and only during first install. It is used to automate setup of Ambari metrics reporting task in Nifi
if not params.is_additional_node:
if params.metrics_collector_host and params.nifi_ambari_reporting_enabled and not sudo.path_isfile(params.nifi_flow_config_dir+'/flow.xml.gz'):
Execute('echo "First time setup so generating flow.xml.gz" >> ' + params.nifi_node_log_file, user=params.nifi_user)
flow_content=InlineTemplate(params.nifi_flow_content)
File(format("{params.nifi_flow_config_dir}/flow.xml"), content=flow_content, owner=params.nifi_user, group=params.nifi_group, mode=0600)
Execute(format("cd {params.nifi_flow_config_dir}; gzip flow.xml;"), user=params.nifi_user)
def setup_keystore_truststore(self, is_starting, params, config_version_file):
if is_starting:
# check against last version to determine if key/trust has changed
last_config_version = nifi_toolkit_util.get_config_version(config_version_file, 'ssl')
last_config = nifi_toolkit_util.get_config_by_version('/var/lib/ambari-agent/data',
'nifi-ambari-ssl-config', last_config_version)
ca_client_dict = nifi_toolkit_util.get_nifi_ca_client_dict(last_config, params)
using_client_json = len(ca_client_dict) == 0 and sudo.path_isfile(
params.nifi_config_dir + '/nifi-certificate-authority-client.json')
if using_client_json:
ca_client_dict = nifi_toolkit_util.load(
params.nifi_config_dir + '/nifi-certificate-authority-client.json')
changed_keystore_truststore = nifi_toolkit_util.changed_keystore_truststore(ca_client_dict,
params.nifi_ca_client_config,
using_client_json) if not len(
ca_client_dict) == 0 else True
if params.nifi_toolkit_tls_regenerate:
nifi_toolkit_util.move_keystore_truststore(ca_client_dict)
ca_client_dict = {}
elif changed_keystore_truststore:
nifi_toolkit_util.move_keystore_truststore(ca_client_dict)
if changed_keystore_truststore or params.nifi_toolkit_tls_regenerate:
nifi_toolkit_util.overlay(ca_client_dict, params.nifi_ca_client_config)
updated_properties = self.run_toolkit_client(ca_client_dict, params.nifi_config_dir, params.jdk64_home,
params.nifi_user, params.nifi_group,
params.stack_support_toolkit_update)
nifi_toolkit_util.update_nifi_properties(updated_properties, params.nifi_properties)
nifi_toolkit_util.save_config_version(config_version_file, 'ssl', params.nifi_ambari_ssl_config_version,
params.nifi_user, params.nifi_group)
elif using_client_json:
nifi_toolkit_util.save_config_version(config_version_file, 'ssl', params.nifi_ambari_ssl_config_version,
params.nifi_user, params.nifi_group)
old_nifi_properties = nifi_toolkit_util.convert_properties_to_dict(
params.nifi_config_dir + '/nifi.properties')
return nifi_toolkit_util.populate_ssl_properties(old_nifi_properties, params.nifi_properties, params)
else:
return params.nifi_properties
def setup_tls_toolkit_upgrade(self,env):
import params
env.set_params(params)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None:
raise Fail('Unable to determine the stack and stack version')
if params.upgrade_direction == Direction.UPGRADE and params.nifi_registry_ssl_enabled and params.nifi_ca_host:
version_file = params.nifi_registry_config_dir + '/config_version'
client_json_file = params.nifi_registry_config_dir+ '/nifi-certificate-authority-client.json'
# if not sudo.path_isfile(version_file):
# Logger.info(format('Create config version file if it does not exist'))
# version_num = params.config['configurationTags']['nifi-registry-ambari-ssl-config']['tag']
# nifi_toolkit_util.save_config_version(version_file,'ssl',version_num,params.nifi_registry_user,params.nifi_registry_group)
if sudo.path_isfile(client_json_file):
Logger.info(format('Remove client json file'))
sudo.unlink(client_json_file)
def action_create(self):
filename = self.resource.filename
comment_symbols = self.resource.comment_symbols
delimiter = self.resource.key_value_delimiter
properties = self.resource.properties
unsaved_values = properties.keys()
new_content_lines = []
if sudo.path_isfile(filename):
file_content = sudo.read_file(filename, encoding=self.resource.encoding)
new_content_lines += file_content.split('\n')
Logger.info(format("Modifying existing properties file: {filename}"))
for line_num in range(len(new_content_lines)):
line = new_content_lines[line_num]
if line.lstrip() and not line.lstrip()[0] in comment_symbols and delimiter in line:
in_var_name = line.split(delimiter)[0].strip()
in_var_value = line.split(delimiter)[1].strip()
if in_var_name in properties:
value = InlineTemplate(unicode(properties[in_var_name])).get_content()
new_content_lines[line_num] = u"{0}{1}{2}".format(unicode(in_var_name), delimiter, value)
unsaved_values.remove(in_var_name)
else:
Logger.info(format("Creating new properties file as {filename} doesn't exist"))
for property_name in unsaved_values:
value = InlineTemplate(unicode(properties[property_name])).get_content()
line = u"{0}{1}{2}".format(unicode(property_name), delimiter, value)
new_content_lines.append(line)
with Environment.get_instance_copy() as env:
File (filename,
content = u"\n".join(new_content_lines) + "\n",
owner = self.resource.owner,
group = self.resource.group,
mode = self.resource.mode,
encoding = self.resource.encoding,
)
def encrypt_sensitive_properties(self, config_version_file, current_version, nifi_config_dir, jdk64_home, nifi_user,
nifi_group, master_key_password, nifi_flow_config_dir, nifi_sensitive_props_key,
is_starting):
Logger.info("Encrypting NiFi sensitive configuration properties")
encrypt_config_script = nifi_toolkit_util.get_toolkit_script('encrypt-config.sh')
encrypt_config_script_prefix = ('JAVA_HOME=' + jdk64_home, encrypt_config_script)
File(encrypt_config_script, mode=0755)
if is_starting:
last_master_key_password = None
last_config_version = nifi_toolkit_util.get_config_version(config_version_file, 'encrypt')
encrypt_config_script_params = ('-v', '-b', nifi_config_dir + '/bootstrap.conf')
encrypt_config_script_params = encrypt_config_script_params + ('-n', nifi_config_dir + '/nifi.properties')
if sudo.path_isfile(nifi_flow_config_dir + '/flow.xml.gz') and len(
sudo.read_file(nifi_flow_config_dir + '/flow.xml.gz')) > 0:
encrypt_config_script_params = encrypt_config_script_params + (
'-f', nifi_flow_config_dir + '/flow.xml.gz', '-s', PasswordString(nifi_sensitive_props_key))
if nifi_toolkit_util.contains_providers(nifi_config_dir + '/login-identity-providers.xml'):
encrypt_config_script_params = encrypt_config_script_params + (
'-l', nifi_config_dir + '/login-identity-providers.xml')
if last_config_version:
last_config = nifi_toolkit_util.get_config_by_version('/var/lib/ambari-agent/data',
'nifi-ambari-config', last_config_version)
last_master_key_password = last_config['configurations']['nifi-ambari-config'][
'nifi.security.encrypt.configuration.password']
if last_master_key_password and last_master_key_password != master_key_password:
encrypt_config_script_params = encrypt_config_script_params + (
'-m', '-w', PasswordString(last_master_key_password))
encrypt_config_script_params = encrypt_config_script_params + ('-p', PasswordString(master_key_password))
encrypt_config_script_prefix = encrypt_config_script_prefix + encrypt_config_script_params
Execute(encrypt_config_script_prefix, user=nifi_user, logoutput=False)
nifi_toolkit_util.save_config_version(config_version_file, 'encrypt', current_version, nifi_user,
nifi_group)
def get_toolkit_script(scriptName,
scriptDir=files_dir,
toolkitDirPrefix='nifi-toolkit-'):
import params
if params.stack_support_nifi_toolkit_package:
result = os.path.join(params.stack_root, 'current', 'nifi-toolkit',
'bin', scriptName)
Logger.info("Using repository toolkit script: " + result)
return result
else:
Logger.info("Trying to find toolkit script dir with prefix: " +
toolkitDirPrefix)
# in case user did not aligned mpack and repo version we can fallback to partly-match toolkit version
# for example DDP=3.2.1.0-111, and toolkit bundled is 3.2.0.0-333, code will check for prefixes
# 3.2.1.0, 3.2.1, 3.2, and as result will use 3.2.0.0-333 toolkit, most likely this will work
prefixes = _get_best_prefixes_match(toolkitDirPrefix)
for prefix in prefixes:
for dir in os.listdir(scriptDir):
if prefix in dir and dir.startswith('nifi-toolkit'):
if prefix != toolkitDirPrefix:
Logger.error(
"Looks like your DDP stack repository not aligned to DDP mpack version, but we found "
+ dir +
" toolkit directory, and will try to use it. If you facing some problems, please, "
+
"align your version as mentioned in documentation."
)
nifiToolkitDir = os.path.join(scriptDir, dir)
result = nifiToolkitDir + '/bin/' + scriptName
if not sudo.path_isfile(result):
raise Exception("Couldn't find file " + result)
Logger.info("Using bundled toolkit script: " + result)
return result
raise Exception("Couldn't find nifi toolkit directory in " + scriptDir)
def encrypt_sensitive_properties(
nifi_config_dir,
jdk64_home,
java_options,
nifi_user,
last_master_key,
master_key_password,
is_starting,
toolkit_tmp_dir,
stack_version_buildnum,
service,
nifi_flow_config_dir=None,
nifi_sensitive_props_key=None,
support_encrypt_authorizers=None,
):
encrypt_config_script = get_toolkit_script('encrypt-config.sh',
toolkit_tmp_dir,
stack_version_buildnum)
encrypt_config_command = (encrypt_config_script, )
environment = {'JAVA_HOME': jdk64_home, 'JAVA_OPTS': java_options}
File(encrypt_config_script, mode=0755)
if is_starting:
if service == NIFI:
Logger.info("Encrypting NiFi sensitive configuration properties")
encrypt_config_command += ('-v', '-b',
nifi_config_dir + '/bootstrap.conf')
encrypt_config_command += ('-n',
nifi_config_dir + '/nifi.properties')
if (sudo.path_isfile(nifi_flow_config_dir + '/flow.xml.gz') and
len(sudo.read_file(nifi_flow_config_dir + '/flow.xml.gz'))
> 0):
encrypt_config_command += (
'-f', nifi_flow_config_dir + '/flow.xml.gz', '-s',
PasswordString(nifi_sensitive_props_key))
if contains_providers(
nifi_config_dir + '/login-identity-providers.xml',
"provider"):
encrypt_config_command += ('-l', nifi_config_dir +
'/login-identity-providers.xml')
if support_encrypt_authorizers and contains_providers(
nifi_config_dir + '/authorizers.xml', "authorizer"):
encrypt_config_command += ('-a', nifi_config_dir +
'/authorizers.xml')
if last_master_key:
encrypt_config_command += ('-m', '-e',
PasswordString(last_master_key))
encrypt_config_command += ('-p',
PasswordString(master_key_password))
Execute(encrypt_config_command,
user=nifi_user,
logoutput=False,
environment=environment)
elif service == NIFI_REGISTRY:
Logger.info(
"Encrypting NiFi Registry sensitive configuration properties")
encrypt_config_command += ('--nifiRegistry', '-v', '-b',
nifi_config_dir + '/bootstrap.conf')
encrypt_config_command += ('-r', nifi_config_dir +
'/nifi-registry.properties')
if contains_providers(nifi_config_dir + '/identity-providers.xml',
"provider"):
encrypt_config_command += ('-i', nifi_config_dir +
'/identity-providers.xml')
if contains_providers(nifi_config_dir + '/authorizers.xml',
"authorizer"):
encrypt_config_command += ('-a', nifi_config_dir +
'/authorizers.xml')
if last_master_key:
encrypt_config_command += ('--oldKey',
PasswordString(last_master_key))
encrypt_config_command += ('-p',
PasswordString(master_key_password))
Execute(encrypt_config_command,
user=nifi_user,
logoutput=False,
environment=environment)
'nifi.node.port']
java_home = config['ambariLevelParams']['java_home']
security_enabled = config['configurations']['cluster-env']['security_enabled']
smokeuser = config['configurations']['cluster-env']['smokeuser']
smokeuser_principal = config['configurations']['cluster-env'][
'smokeuser_principal_name']
smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
kinit_path_local = get_kinit_path(
default('/configurations/kerberos-env/executable_search_paths', None))
stack_support_nifi_toolkit_package = check_stack_feature(
'nifi_toolkit_package', version_for_stack_feature_checks)
#some released HDP stacks will not have this stack feature, manually check
if not stack_support_nifi_toolkit_package and stack_name == "HDP":
marker_script = os.path.join(stack_root,
"current/nifi-toolkit/bin/tls-toolkit.sh")
if sudo.path_isfile(marker_script):
stack_support_nifi_toolkit_package = True
stack_support_encrypt_config = check_stack_feature(
'nifi_encrypt_config', version_for_stack_feature_checks)
stack_support_toolkit_update = check_stack_feature(
'toolkit_config_update', version_for_stack_feature_checks)
stack_support_admin_toolkit = check_stack_feature(
'admin_toolkit_support', version_for_stack_feature_checks)
stack_support_nifi_jaas = check_stack_feature(
'nifi_jaas_conf_create', version_for_stack_feature_checks)
stack_support_encrypt_authorizers = check_stack_feature(
'nifi_encrypted_authorizers_config', version_for_stack_feature_checks)
stack_support_nifi_auto_client_registration = check_stack_feature(
'nifi_auto_client_registration', version_for_stack_feature_checks)
if security_enabled:
def store_exists(client_dict, key):
if key not in client_dict:
return False
return sudo.path_isfile(client_dict[key])
def load(config_json):
if sudo.path_isfile(config_json):
contents = sudo.read_file(config_json)
if len(contents) > 0:
return json.loads(contents)
return {}