def action_create(self):
path = self.resource.path
if sudo.path_lexists(path):
oldpath = os.path.realpath(path)
if oldpath == self.resource.to:
return
if not sudo.path_lexists(path):
raise Fail(
"%s trying to create a symlink with the same name as an existing file or directory" % self.resource)
Logger.info("%s replacing old symlink to %s" % (self.resource, oldpath))
sudo.unlink(path)
if self.resource.hard:
if not sudo.path_exists(self.resource.to):
raise Fail("Failed to apply %s, linking to nonexistent location %s" % (self.resource, self.resource.to))
if sudo.path_isdir(self.resource.to):
raise Fail("Failed to apply %s, cannot create hard link to a directory (%s)" % (self.resource, self.resource.to))
Logger.info("Creating hard %s" % self.resource)
sudo.link(self.resource.to, path)
else:
if not sudo.path_exists(self.resource.to):
Logger.info("Warning: linking to nonexistent location %s" % self.resource.to)
Logger.info("Creating symbolic %s to %s" % (self.resource, self.resource.to))
sudo.symlink(self.resource.to, path)
def action_create(self):
path = self.resource.path
if sudo.path_lexists(path):
oldpath = os.path.realpath(path)
if oldpath == self.resource.to:
return
if not sudo.path_lexists(path):
raise Fail(
"%s trying to create a symlink with the same name as an existing file or directory"
% self.resource)
Logger.info("%s replacing old symlink to %s" %
(self.resource, oldpath))
sudo.unlink(path)
if self.resource.hard:
if not sudo.path_exists(self.resource.to):
raise Fail(
"Failed to apply %s, linking to nonexistent location %s" %
(self.resource, self.resource.to))
if sudo.path_isdir(self.resource.to):
raise Fail(
"Failed to apply %s, cannot create hard link to a directory (%s)"
% (self.resource, self.resource.to))
Logger.info("Creating hard %s" % self.resource)
sudo.link(self.resource.to, path)
else:
if not sudo.path_exists(self.resource.to):
Logger.info("Warning: linking to nonexistent location %s" %
self.resource.to)
Logger.info("Creating symbolic %s to %s" %
(self.resource, self.resource.to))
sudo.symlink(self.resource.to, path)
def invalidate_ca_server(self, env):
import params
ca_json = os.path.join(params.nifi_config_dir,
'nifi-certificate-authority.json')
nifi_toolkit_util_common.move_store(
nifi_toolkit_util_common.load(ca_json), 'keyStore')
unlink(ca_json)
def remove_symlinks(stack_location, service_definitions_location,
staged_mpack_dir):
"""
Helper function to remove all symbolic links pointed to a management pack
:param stack_location: Path to stacks folder
(/var/lib/ambari-server/resources/stacks)
:param service_definitions_location: Path to service_definitions folder
(/var/lib/ambari-server/resources/common-services)
:param staged_mpack_dir: Path to management pack staging location
(/var/lib/ambari-server/resources/mpacks/mpack_name-mpack_version)
"""
for location in [stack_location, service_definitions_location]:
for root, dirs, files in os.walk(location):
for name in files:
file = os.path.join(root, name)
if os.path.islink(
file) and staged_mpack_dir in os.path.realpath(file):
print_info_msg("Removing symlink {0}".format(file))
sudo.unlink(file)
for name in dirs:
dir = os.path.join(root, name)
if os.path.islink(
dir) and staged_mpack_dir in os.path.realpath(dir):
print_info_msg("Removing symlink {0}".format(dir))
sudo.unlink(dir)
def process_stack_definition_artifact(artifact, artifact_source_dir, options):
"""
Process stack-definition artifact
:param artifact: Artifact metadata
:param artifact_source_dir: Location of artifact in the management pack
:param options: Command line options
"""
# Get ambari mpack properties
stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties(
)
stack_name = None
if "stack_name" in artifact:
stack_name = artifact.stack_name
if not stack_name:
print_error_msg(
"Must provide stack name for stack-definition artifact!")
raise FatalException(
-1, 'Must provide stack name for stack-definition artifact!')
stack_version = None
if "stack_version" in artifact:
stack_version = artifact.stack_version
if not stack_version:
print_error_msg(
"Must provide stack version for stack-definition artifact!")
raise FatalException(
-1, 'Must provide stack version for stack-definition artifact!')
dest_link = os.path.join(stack_location, stack_name, stack_version)
if options.force and os.path.islink(dest_link):
sudo.unlink(dest_link)
sudo.symlink(artifact_source_dir, dest_link)
def process_stack_extension_definition_artifact(artifact, artifact_source_dir, options):
"""
Process stack-extension-definition artifact
:param artifact: Artifact metadata
:param artifact_source_dir: Location of artifact in the management pack
:param options: Command line options
"""
# Get ambari mpack properties
stack_location, service_definitions_location, mpacks_staging_location = get_mpack_properties()
service_name = None
if "service_name" in artifact:
service_name = artifact.service_name
if not service_name:
print_error_msg("Must provide service name for stack-extension-definition artifact!")
raise FatalException(-1, 'Must provide service name for stack-extension-definition artifact!')
applicable_stacks = None
if "applicable_stacks" in artifact:
applicable_stacks = artifact.applicable_stacks
if not applicable_stacks:
print_error_msg("Must provide applicable stacks for stack-extension-definition artifact!")
raise FatalException(-1, 'Must provide applicable stacks for stack-extension-definition artifact!')
for applicable_stack in applicable_stacks:
stack_name = applicable_stack.stack_name
stack_version = applicable_stack.stack_version
dest_stack_path = os.path.join(stack_location, stack_name)
dest_stack_version_path = os.path.join(dest_stack_path, stack_version)
dest_stack_services_path = os.path.join(dest_stack_version_path, "services")
dest_link = os.path.join(dest_stack_services_path, service_name)
if os.path.exists(dest_stack_path) and os.path.exists(dest_stack_version_path):
if not os.path.exists(dest_stack_services_path):
sudo.makedir(dest_stack_services_path, 0755)
if options.force and os.path.islink(dest_link):
sudo.unlink(dest_link)
sudo.symlink(artifact_source_dir, dest_link)
def stop(self, env, upgrade_type=None):
import params
import status_params
env.set_params(params)
env.set_params(status_params)
# this method will be called during an upgrade before start/configure get to setup all the permissions so we need to do it here too
Directory([params.bin_dir],
owner=params.nifi_registry_user,
group=params.nifi_registry_group,
create_parents=True,
recursive_ownership=True,
cd_access='a')
env_content = InlineTemplate(params.nifi_registry_env_content)
File(format("{params.bin_dir}/nifi-registry-env.sh"),
content=env_content,
owner=params.nifi_registry_user,
group=params.nifi_registry_group,
mode=0755)
Execute('export JAVA_HOME=' + params.jdk64_home + ';' +
params.bin_dir + '/nifi-registry.sh stop >> ' +
params.nifi_registry_log_file,
user=params.nifi_registry_user)
if os.path.isfile(status_params.nifi_registry_pid_file):
sudo.unlink(status_params.nifi_registry_pid_file)
def stop(self, env, upgrade_type=None):
import params
import status_params
env.set_params(params)
env.set_params(status_params)
Directory([params.nifi_registry_bin_dir],
owner=params.nifi_registry_user,
group=params.nifi_registry_group,
create_parents=True,
recursive_ownership=True,
cd_access='a')
env_content = InlineTemplate(params.nifi_registry_env_content)
File(format("{params.nifi_registry_bin_dir}/nifi-registry-env.sh"),
content=env_content,
owner=params.nifi_registry_user,
group=params.nifi_registry_group,
mode=0755)
Execute('export JAVA_HOME=' + params.jdk64_home + ';' +
params.nifi_registry_bin_dir + '/nifi-registry.sh stop >> ' +
params.nifi_registry_log_file,
user=params.nifi_registry_user)
if os.path.isfile(status_params.nifi_registry_pid_file):
sudo.unlink(status_params.nifi_registry_pid_file)
def stop(self, env, upgrade_type=None):
import params
import status_params
nifi_toolkit_util_common.copy_toolkit_scripts(
params.toolkit_files_dir,
params.toolkit_tmp_dir,
params.nifi_user,
params.nifi_group,
upgrade_type,
service=nifi_toolkit_util_common.NIFI)
run_ca_script = os.path.join(params.toolkit_tmp_dir, 'run_ca.sh')
ca_server_script = nifi_toolkit_util_common.get_toolkit_script(
'tls-toolkit.sh', params.toolkit_tmp_dir,
params.stack_version_buildnum)
File(ca_server_script, mode=0755)
File(run_ca_script, mode=0755)
if path_isfile(status_params.nifi_ca_pid_file):
Execute(
(run_ca_script, "stop", params.jdk64_home, ca_server_script,
params.nifi_config_dir + '/nifi-certificate-authority.json',
params.nifi_ca_log_file_stdout,
params.nifi_ca_log_file_stderr,
status_params.nifi_ca_pid_file, params.toolkit_ca_check_url),
user=params.nifi_user,
logoutput=True)
try:
self.status(env)
except ComponentIsNotRunning:
unlink(status_params.nifi_ca_pid_file)
def move_store(client_dict, key):
if store_exists(client_dict, key):
num = 0
name = client_dict[key]
while sudo.path_isfile(name + '.bak.' + str(num)):
num += 1
sudo.copy(name, name + '.bak.' + str(num))
sudo.unlink(name)
def action_delete(self):
path = self.resource.path
if sudo.path_isdir(path):
raise Fail("Applying %s failed, %s is directory not file!" % (self.resource, path))
if sudo.path_exists(path):
Logger.info("Deleting %s" % self.resource)
sudo.unlink(path)
def action_delete(self):
path = self.resource.path
if sudo.path_isdir(path):
raise Fail("Applying %s failed, %s is directory not file!" % (self.resource, path))
if sudo.path_exists(path):
Logger.info("Deleting %s" % self.resource)
sudo.unlink(path)
def stop(self, env):
import params
import status_params
Execute('export JAVA_HOME=' + params.jdk64_home + ';' +
params.bin_dir + '/nifi.sh stop >> ' +
params.nifi_node_log_file,
user=params.nifi_user)
if os.path.isfile(status_params.nifi_node_pid_file):
sudo.unlink(status_params.nifi_node_pid_file)
def stop(self, env):
# Import properties defined in -env.xml file from the status_params class
import status_params
# Stop Kibana
cmd = 'systemctl stop kibana.service'
Execute(cmd)
if os.path.isfile(status_params.kibana_pid_file):
sudo.unlink(status_params.kibana_pid_file)
print 'Stop kibana complete'
def create_symlink_using_path(src_path, dest_link, force=False):
"""
Helper function to create symbolic link (dest_link -> src_path)
:param src_path: Source path
:param dest_link: Destination link
:param force: Remove existing symlink
"""
if force and os.path.islink(dest_link):
sudo.unlink(dest_link)
sudo.symlink(src_path, dest_link)
print_info_msg("Symlink: " + dest_link)
def remove_config_version(version_file, version_type, nifi_user, nifi_group):
if sudo.path_isfile(version_file):
contents = sudo.read_file(version_file)
version = json.loads(contents)
version.pop(version_type, None)
sudo.unlink(version_file)
File(version_file,
owner=nifi_user,
group=nifi_group,
mode=0600,
content=json.dumps(version))
def stop(self, env, upgrade_type=None):
import params
import status_params
env.set_params(params)
env.set_params(status_params)
env_content=InlineTemplate(params.nifi_env_content)
File(format("{params.bin_dir}/nifi-env.sh"), content=env_content, owner=params.nifi_user, group=params.nifi_group, mode=0755)
Execute ('export JAVA_HOME='+params.jdk64_home+';'+params.bin_dir+'/nifi.sh stop >> ' + params.nifi_node_log_file, user=params.nifi_user)
if os.path.isfile(status_params.nifi_node_pid_file):
sudo.unlink(status_params.nifi_node_pid_file)
def stop(self, env):
# Import properties defined in -env.xml file from the status_params class
import status_params
# Stop Filebeat
cmd = '/etc/init.d/filebeat stop'
Execute(cmd)
if os.path.isfile(status_params.filebeat_pid_file):
sudo.unlink(status_params.filebeat_pid_file)
print 'Stop filebeat complete'
def create_symlink(src_dir, dest_dir, file_name, force=False):
"""
Helper function to create symbolic link (dest_dir/file_name -> src_dir/file_name)
:param src_dir: Source directory
:param dest_dir: Destination directory
:param file_name: File name
:param force: Remove existing symlink
"""
src_path = os.path.join(src_dir, file_name)
dest_link = os.path.join(dest_dir, file_name)
if force and os.path.islink(dest_link):
sudo.unlink(dest_link)
sudo.symlink(src_path, dest_link)
def process_stack_addon_service_definitions_artifact(artifact,
artifact_source_dir,
options):
"""
Process stack addon service definitions artifact
:param artifact: Artifact metadata
:param artifact_source_dir: Location of artifact in the management pack
:param options: Command line options
"""
# Get ambari mpack properties
stack_location, extension_location, service_definitions_location, mpacks_staging_location, dashboard_location = get_mpack_properties(
)
service_versions_map = None
if "service_versions_map" in artifact:
service_versions_map = artifact.service_versions_map
if not service_versions_map:
msg = "Must provide service versions map for " + STACK_ADDON_SERVICE_DEFINITIONS_ARTIFACT_NAME + " artifact!"
print_error_msg(msg)
raise FatalException(-1, msg)
for service_name in sorted(os.listdir(artifact_source_dir)):
source_service_path = os.path.join(artifact_source_dir, service_name)
for service_version in sorted(os.listdir(source_service_path)):
source_service_version_path = os.path.join(source_service_path,
service_version)
for service_version_entry in service_versions_map:
if service_name == service_version_entry.service_name and service_version == service_version_entry.service_version:
applicable_stacks = service_version_entry.applicable_stacks
for applicable_stack in applicable_stacks:
stack_name = applicable_stack.stack_name
stack_version = applicable_stack.stack_version
dest_stack_path = os.path.join(stack_location,
stack_name)
dest_stack_version_path = os.path.join(
dest_stack_path, stack_version)
dest_stack_services_path = os.path.join(
dest_stack_version_path, SERVICES_DIRNAME)
dest_link = os.path.join(dest_stack_services_path,
service_name)
if os.path.exists(dest_stack_path) and os.path.exists(
dest_stack_version_path):
if not os.path.exists(dest_stack_services_path):
sudo.makedir(dest_stack_services_path, 0755)
if options.force and os.path.islink(dest_link):
sudo.unlink(dest_link)
sudo.symlink(source_service_version_path,
dest_link)
create_dashboard_symlinks(
source_service_version_path, service_name,
dashboard_location, options)
def setup_tls_toolkit_upgrade(self, env):
import params
env.set_params(params)
if params.upgrade_direction == Direction.UPGRADE and params.nifi_ssl_enabled and params.nifi_ca_host:
version_file = params.nifi_config_dir + '/config_version'
client_json_file = params.nifi_config_dir + '/nifi-certificate-authority-client.json'
if sudo.path_isfile(version_file):
Logger.info(format('Remove config version file if does exist'))
sudo.unlink(version_file)
if sudo.path_isfile(client_json_file):
Logger.info(format('Remove client json file'))
sudo.unlink(client_json_file)
def setup_tls_toolkit_upgrade(self, env):
import params
env.set_params(params)
version_file = params.nifi_config_dir + '/config_version'
client_json_file = params.nifi_config_dir + '/nifi-certificate-authority-client.json'
if not sudo.path_isfile(version_file):
Logger.info(format('Create config version file if it does not exist'))
version_num = params.config['configurationTags']['nifi-ambari-ssl-config']['tag']
nifi_toolkit_util.save_config_version(version_file, 'ssl', version_num, params.nifi_user,
params.nifi_group)
if sudo.path_isfile(client_json_file):
Logger.info(format('Remove client json file'))
sudo.unlink(client_json_file)
def save_config_version(version_file, version_type, version_num,
nifi_registry_user, nifi_registry_group):
version = {}
if sudo.path_isfile(version_file):
contents = sudo.read_file(version_file)
version = json.loads(contents)
version[version_type] = version_num
sudo.unlink(version_file)
else:
version[version_type] = version_num
File(version_file,
owner=nifi_registry_user,
group=nifi_registry_group,
mode=0600,
content=json.dumps(version))
def stop(self, env):
import status_params
if path_isfile(status_params.nifi_ca_pid_file):
try:
self.status(env)
pid = int(read_file(status_params.nifi_ca_pid_file))
for i in range(25):
kill(pid, SIGTERM)
time.sleep(1)
self.status(env)
kill(pid, SIGKILL)
time.sleep(5)
self.status(env)
except ComponentIsNotRunning:
unlink(status_params.nifi_ca_pid_file)
def stop(self, env):
# Import properties defined in -config.xml file from the params class
import params
# Import properties defined in -env.xml file from the status_params class
import status_params
# This allows us to access the params.elastic_pid_file property as
# format('{elastic_pid_file}')
env.set_params(params)
# Stop Elasticsearch
Execute("service elasticsearch stop")
if os.path.isfile(status_params.elastic_pid_file):
# kill_process(params.elastic_pid_file, params.elastic_user, params.elastic_log_dir)
sudo.unlink(status_params.elastic_pid_file)
def setup_tls_toolkit_upgrade(self,env):
import params
env.set_params(params)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None:
raise Fail('Unable to determine the stack and stack version')
if params.upgrade_direction == Direction.UPGRADE and params.nifi_ssl_enabled and params.nifi_ca_host:
version_file = params.nifi_config_dir + '/config_version'
client_json_file = params.nifi_config_dir+ '/nifi-certificate-authority-client.json'
if sudo.path_isfile(version_file):
Logger.info(format('Remove config version file if does exist'))
sudo.unlink(version_file)
if sudo.path_isfile(client_json_file):
Logger.info(format('Remove client json file'))
sudo.unlink(client_json_file)
def setup_tls_toolkit_upgrade(self,env):
import params
env.set_params(params)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None:
raise Fail('Unable to determine the stack and stack version')
if params.upgrade_direction == Direction.UPGRADE and params.nifi_registry_ssl_enabled and params.nifi_ca_host:
version_file = params.nifi_registry_config_dir + '/config_version'
client_json_file = params.nifi_registry_config_dir+ '/nifi-certificate-authority-client.json'
# if not sudo.path_isfile(version_file):
# Logger.info(format('Create config version file if it does not exist'))
# version_num = params.config['configurationTags']['nifi-registry-ambari-ssl-config']['tag']
# nifi_toolkit_util.save_config_version(version_file,'ssl',version_num,params.nifi_registry_user,params.nifi_registry_group)
if sudo.path_isfile(client_json_file):
Logger.info(format('Remove client json file'))
sudo.unlink(client_json_file)
def main():
properties = get_ambari_properties()
if properties == -1:
print >> sys.stderr, "Error getting ambari properties"
return -1
resources_location = get_resources_location(properties)
views_dir = os.path.join(resources_location, "views")
for file in os.listdir(views_dir):
path = os.path.join(views_dir, file)
if os.path.isfile(path):
if "ambari-admin" in path or "storm-view" in path:
print "Keeping views jar : " + path
else:
print "Deleting views jar : " + path
sudo.unlink(path)
else:
print "Deleting views directory : " + path
sudo.rmtree(path)
return 0
def create_30_config_version(self, env):
package_name = 'registry'
stack_root = Script.get_stack_root()
current_dir = "{0}/current/registry/conf".format(stack_root)
directories = [{
"conf_dir": "/etc/registry/conf",
"current_dir": current_dir
}]
stack_version = stack_select.get_stack_version_before_install(
package_name)
conf_dir = "/etc/registry/conf"
if stack_version:
try:
#Check if broken symbolic links issue exists
os.stat(conf_dir)
conf_select.convert_conf_directories_to_symlinks(
package_name, stack_version, directories)
cp_cmd = as_sudo([
"cp", "-a", "-f", "/etc/registry/conf.backup/.",
"/etc/registry/conf"
])
Execute(cp_cmd, logoutput=True)
except OSError as e:
Logger.warning(
"Detected broken symlink : {0}. Attempting to repair.".
format(str(e)))
#removing symlink conf directory
sudo.unlink(conf_dir)
#make conf dir again
sudo.makedirs(conf_dir, 0755)
#copy all files
for files in glob.glob("/etc/registry/conf.backup/*"):
cp_cmd = as_sudo(["cp", "-r", files, conf_dir])
Execute(cp_cmd, logoutput=True)
conf_select.convert_conf_directories_to_symlinks(
package_name, stack_version, directories)
def action_delete(self):
path = self.resource.path
if sudo.path_lexists(path):
Logger.info("Deleting %s" % self.resource)
sudo.unlink(path)
def action_delete(self):
path = self.resource.path
if sudo.path_lexists(path):
Logger.info("Deleting %s" % self.resource)
sudo.unlink(path)
