def start(self, env, upgrade_type=None):
import params
import status_params
install_nifi()
install_nifi_toolkit()
nifi_toolkit_util_common.copy_toolkit_scripts(
params.toolkit_files_dir,
params.toolkit_tmp_dir,
params.nifi_user,
params.nifi_group,
upgrade_type,
service=nifi_toolkit_util_common.NIFI)
self.configure(env)
ca_server_script = nifi_toolkit_util_common.get_toolkit_script(
'tls-toolkit.sh', params.toolkit_tmp_dir,
params.stack_version_buildnum)
run_ca_script = os.path.join(params.toolkit_tmp_dir, 'run_ca.sh')
Directory([params.nifi_config_dir],
owner=params.nifi_user,
group=params.nifi_group,
create_parents=True,
recursive_ownership=True)
File(ca_server_script, mode=0755)
File(run_ca_script, mode=0755)
Execute(
(run_ca_script, "start", params.jdk64_home, ca_server_script,
params.nifi_config_dir + '/nifi-certificate-authority.json',
params.nifi_ca_log_file_stdout, params.nifi_ca_log_file_stderr,
status_params.nifi_ca_pid_file, params.toolkit_ca_check_url),
user=params.nifi_user,
logoutput=True)
if not os.path.isfile(status_params.nifi_ca_pid_file):
raise Exception('Expected pid file to exist')
def stop(self, env, upgrade_type=None):
import params
import status_params
nifi_toolkit_util_common.copy_toolkit_scripts(
params.toolkit_files_dir,
params.toolkit_tmp_dir,
params.nifi_user,
params.nifi_group,
upgrade_type,
service=nifi_toolkit_util_common.NIFI)
run_ca_script = os.path.join(params.toolkit_tmp_dir, 'run_ca.sh')
ca_server_script = nifi_toolkit_util_common.get_toolkit_script(
'tls-toolkit.sh', params.toolkit_tmp_dir,
params.stack_version_buildnum)
File(ca_server_script, mode=0755)
File(run_ca_script, mode=0755)
if path_isfile(status_params.nifi_ca_pid_file):
Execute(
(run_ca_script, "stop", params.jdk64_home, ca_server_script,
params.nifi_config_dir + '/nifi-certificate-authority.json',
params.nifi_ca_log_file_stdout,
params.nifi_ca_log_file_stderr,
status_params.nifi_ca_pid_file, params.toolkit_ca_check_url),
user=params.nifi_user,
logoutput=True)
try:
self.status(env)
except ComponentIsNotRunning:
unlink(status_params.nifi_ca_pid_file)
def check_nifi_registry_portal_with_toolkit(url, jdk64_home,
nifi_registry_dir,
nifi_registry_bootstrap,
toolkit_tmp_dir,
stack_version_buildnum):
Logger.info("Checking Nifi Registry portal with toolkit")
tls_toolkit_script = nifi_toolkit_util_common.get_toolkit_script(
'tls-toolkit.sh', toolkit_tmp_dir, stack_version_buildnum)
File(tls_toolkit_script, mode=0755)
nifi_registry_props_file = nifi_registry_dir + '/conf/nifi-registry.properties'
nifi_registry_props = NifiRegistryServiceCheck.convert_properties_to_dict(
jdk64_home, nifi_registry_props_file, nifi_registry_bootstrap,
toolkit_tmp_dir, stack_version_buildnum)
if len(nifi_registry_props) == 0:
raise Fail('Unable to read properties from {0}'.format(
nifi_registry_props_file))
else:
keystore = nifi_registry_props['nifi.registry.security.keystore']
keystoreType = nifi_registry_props[
'nifi.registry.security.keystoreType']
keystorePasswd = nifi_registry_props[
'nifi.registry.security.keystorePasswd']
keyPasswd = nifi_registry_props['nifi.registry.security.keyPasswd']
truststore = nifi_registry_props[
'nifi.registry.security.truststore']
truststoreType = nifi_registry_props[
'nifi.registry.security.truststoreType']
truststorePasswd = nifi_registry_props[
'nifi.registry.security.truststorePasswd']
command = 'ambari-sudo.sh JAVA_HOME=' + jdk64_home + ' ' + tls_toolkit_script + ' status -u ' + url + ' -ks ' + keystore + ' -kst ' + keystoreType + ' -ksp ' + keystorePasswd + ' -kp ' + keyPasswd + ' -ts ' + truststore + ' -tst ' + truststoreType + ' -tsp ' + truststorePasswd
# Only uncomment for debugging, otherwise the passwords will get logged
# Logger.info("Executing: " + command)
code, out = shell.call(command, quiet=True, logoutput=False)
if code > 0:
raise Fail(
"Call to tls-toolkit encountered error: {0}".format(out))
else:
if out.find('Error communicating with') > -1:
raise Fail(
"Error connecting to NiFi Registry: {0}".format(out))
def convert_properties_to_dict(jdk64_home, nifi_registry_props_file, nifi_registry_bootstrap, toolkit_tmp_dir, stack_version_buildnum):
dict = {}
if sudo.path_isfile(nifi_registry_props_file):
encrypt_tool_script = nifi_toolkit_util_common.get_toolkit_script('encrypt-config.sh', toolkit_tmp_dir, stack_version_buildnum)
File(encrypt_tool_script, mode=0755)
command = 'ambari-sudo.sh JAVA_HOME=' + jdk64_home + ' '+ encrypt_tool_script + ' --nifiRegistry --decrypt -r ' + nifi_registry_props_file + ' -b ' + nifi_registry_bootstrap
code, out = shell.call(command,quiet=True,logoutput=False)
lines = out.split('\n')
for line in lines:
props = line.rstrip().split('=')
if len(props) == 2:
dict[props[0]] = props[1]
elif len(props) == 1:
dict[props[0]] = ''
return dict
def check_nifi_portal_with_toolkit(urls, jdk64_home, nifi_dir,
nifi_bootstrap, toolkit_tmp_dir,
stack_version_buildnum):
node_manager_script = nifi_toolkit_util_common.get_toolkit_script(
'node-manager.sh', toolkit_tmp_dir, stack_version_buildnum)
File(node_manager_script, mode=0755)
command = 'ambari-sudo.sh JAVA_HOME=' + jdk64_home + ' ' + node_manager_script + ' -d ' + nifi_dir + ' -b ' + nifi_bootstrap + ' -o status -u "' + ','.join(
urls) + '"'
code, out = shell.call(command, quiet=True, logoutput=False)
if code > 0:
raise Fail(
"Call to admin-toolkit encountered error: {0}".format(out))
else:
if out.find('did not complete due to exception') > -1:
raise Fail(
"Error connecting to one or more nifi nodes: {0}".format(
out))
def nifi_cli(command=None,
subcommand=None,
errors_retries=12,
retries_pause=10,
acceptable_errors=None,
**command_args):
"""
Executes nifi cli command and returns its output.
We need execute command several times because nifi becomes responsive after some among of time.
On non super-fast vm it takes 1.5 minutes to get nifi responding for cli calls.
Also some commands can produce different errors but after some time that errors disappear.
In other works - this cli is hard to use in automated environments :).
:param command: main cli command(nifi, registry, session, etc)
:param subcommand: sub-command of main command(nifi list-reg-clients, etc)
:param errors_retries: retries count on acceptable errors
:param retries_pause: pause between call retries
:param acceptable_errors: errors that is acceptable for retry("Connection refused" error always in this list)
:param command_args: long version of command parameters
:return: command output
"""
import params
cli_env = {"JAVA_HOME": params.java_home}
cli_script = nifi_toolkit_util_common.get_toolkit_script(
"cli.sh", params.toolkit_tmp_dir, params.stack_version_buildnum)
if errors_retries < 1:
errors_retries = 1
if acceptable_errors is None:
acceptable_errors = []
acceptable_errors.append("Connection refused")
def do_retry(output):
for acceptable_error in acceptable_errors:
if acceptable_error in output:
return True
return False
cmd = [cli_script, command]
if subcommand is not None:
cmd.append(subcommand)
client_opts = nifi_toolkit_util_common.get_client_opts()
if params.nifi_ssl_enabled:
command_args.update(nifi_toolkit_util_common.get_client_opts())
command_args["proxiedEntity"] = params.nifi_initial_admin_id
else:
command_args["baseUrl"] = client_opts["baseUrl"]
for arg_name, arg_value in command_args.iteritems():
cmd.append("--" + arg_name)
cmd.append(arg_value)
for _ in range(0, errors_retries):
errors_retries -= 1
code, out = shell.call(cmd,
sudo=True,
env=cli_env,
logoutput=False,
quiet=True)
if code != 0 and do_retry(out) and errors_retries != 0:
time.sleep(retries_pause)
continue
elif code == 0:
return out
else:
raise Fail("Failed to execute nifi cli.sh command")