def create_host_directories_and_tar(self):
"""Main packaging function
Works in 3 parts:
1. Validate app data and configurations
2. Create tmp directories for each host with loaded apps and manifest
3. Package (tar) up host tmp directories for distribution
"""
Helpers.delete_path(self.tmp_folder)
Helpers.create_path(self.tars_folder, True)
self.repo_manager.set_commit_id()
master_commit_log = self.repo_manager.get_commit_log()
errors_found = False
changes_found = False
for host in self.appetite_hosts: # pylint: disable=too-many-nested-blocks
# Per host build apps folder and tar up based on class
hostname = host.hostname
apps = host.get_apps(self.args.refname)
tarname = host.tarname
apps = sorted(apps, key=lambda app: app.commit_id)
tmp_hostname_dir = os.path.join(self.hosts_folder, hostname)
tmp_hostname_meta = os.path.join(tmp_hostname_dir, Consts.META_DIR)
apps_meta = []
if len(apps) < 1:
Logger.warn("Host with no apps", hostname=hostname)
continue
# Parse the remote meta file from the host
# This file might not exist
remote_meta_file = host.local_meta_file
remote_metas_loaded = False
if os.path.exists(remote_meta_file):
try:
with open(remote_meta_file) as remote_data_file:
remote_metas_master = json.load(remote_data_file)
remote_metas_content = remote_metas_master['content'] \
if 'content' in remote_metas_master else remote_metas_master
remote_metas = [
AppetiteHost.create_app_from_object(
self.repo_manager, self.deployment_manager,
meta_data)
for meta_data in remote_metas_content
]
remote_metas_loaded = True
except Exception as exception:
Logger.error("Problems loading meta file",
error=exception.message,
path=remote_meta_file)
elif not self.args.dryrun:
Logger.warn("Local version of remote meta not found",
file=remote_meta_file)
ordered_unique_apps = sorted(list(set(apps)),
key=lambda single_app:
(single_app.name, single_app.
commit_id, single_app.method_name))
for iapp in ordered_unique_apps:
app_occurrences = apps.count(iapp)
if app_occurrences > 1:
Logger.warn("Dup app found",
host=host.hostname,
app_info=iapp.app_key,
occurences=app_occurrences)
# Validate app data and configurations
# Go through the apps and checks to see if there are any errors
# This is where the remote meta is compared to the newly generated
# lists of apps from the manifest
for app in apps:
raw_app_path = os.path.join(self.apps_folder, app.name)
# Check the commit Id for problems
if app.commit_id:
self.repo_manager.set_commit_id(app.commit_id)
else: # pylint: disable=else-if-used
if self.args.strict_commitids:
Logger.error("Application with missing commit Id",
hostname=hostname,
app=app.name)
errors_found = True
continue
else:
app._commit_id = master_commit_log['app_commit_id'] # pylint: disable=protected-access
self.repo_manager.set_commit_id(app.commit_id)
# Checks if app listed in the manifest
# exists with the correct commit id
if Helpers.check_path(raw_app_path):
meta_to_append = None
app.refresh_version_info(self.args.refname,
Consts.META_APP_UNCHANGED)
remote_meta = None
# Check to see what has changed
if remote_metas_loaded:
# Searches remote meta to see if application already exists
remote_meta = next((rmeta for rmeta in remote_metas
if app.check_names(rmeta)), None)
if remote_meta:
# If app does exist on system, have the commit ids changed
if remote_meta.commit_id != app.commit_id:
meta_to_append = app.set_status_changed()
else:
# meta has not changed so use existing meta
meta_to_append = app.clone
meta_to_append.update_app_version(app)
# to track if an app is removed from the remote meta
remote_metas.remove(remote_meta)
if not meta_to_append:
# There is no remote meta so all files should be added
meta_to_append = app.set_status_added()
if remote_meta and meta_to_append:
meta_outcome = Helpers.debug_app_versions(
meta_to_append, remote_meta, meta_to_append.status)
Logger.debug("Check meta logic", outcome=meta_outcome)
if meta_to_append.has_changed:
Logger.info("App change", logic=meta_outcome)
apps_meta.append(meta_to_append)
else:
Logger.error("Missing application",
hostname=hostname,
app=app.name,
path=raw_app_path)
continue
if remote_metas_loaded and len(remote_metas) > 0:
# Any apps left in the remote meta do not exist in the current
# manifest and should be deleted
delete_list = []
for deleted_app in remote_metas:
if deleted_app.method_info:
deleted_app.set_status_deleted()
# Added logic check to catch method changes
added_app_found = next((
app for app in apps_meta
if app.status == Consts.META_APP_ADDED and app.name
== deleted_app.name and app.method_info['path'] ==
deleted_app.method_info['path']), None)
if added_app_found:
added_app_found.set_status_changed()
else:
delete_list.append(deleted_app)
else:
Logger.error(
"Problems with method info for deleted app.",
hostname=hostname,
app=deleted_app.name)
apps_meta += delete_list
# Only do something if there has been a change
if len([app for app in apps_meta if not app.is_unchanged]) < 1:
continue
# No point continuing if there is no connection to the host
if not self.check_host_connection(host):
continue
# Clean command lines for auth params
# This data is ingested so creds should be removed
# apps_meta = [updated_app.clone for updated_app in apps_meta]
if not self.args.disable_logging:
for updated_app in apps_meta:
Logger.log_event(updated_app.to_dict)
# Applications that actually needs to be updated
tar_apps = sorted([
updated_app for updated_app in apps_meta if updated_app.updated
],
key=lambda tar_app: tar_app.app)
use_templating = self.template_values and self.args.templating
# Checking will allow templating otherwise will skip steps
Helpers.create_path(
os.path.join(tmp_hostname_meta, Consts.HOST_LOGS_FOLDER_NAME),
True)
if len(tar_apps) > 0:
# All error checks have been done above, build out
# the hosts directory and tar up
for updated_app in tar_apps:
app_path = os.path.join(tmp_hostname_dir,
updated_app.method_info['path'])
Helpers.create_path(app_path, True)
raw_app_path = os.path.join(self.apps_folder,
updated_app.name)
self.repo_manager.set_commit_id(updated_app.commit_id)
if updated_app.update_method_is_copy:
app_dest = os.path.join(app_path,
updated_app.app_clean)
else:
app_dest = app_path
copy_tree(raw_app_path, app_dest)
lookups_inclusion_location = os.path.join(
app_dest, self.deployment_manager.inclusion_filename)
ignore_dir = os.path.join(app_dest, Consts.TMP_IGNORE_DIR)
# Ignore files/folders set in the global configurations
if self.args.install_ignore:
content_ignored_results = Helpers.move_regexed_files(
self.args.install_ignore.split(';'), app_dest,
ignore_dir)
files_included = content_ignored_results['files_moved']
if len(files_included) > 0:
Logger.error(
"Globally these files should not exist in the App. "
"The files have been removed from the install.",
files=files_included,
hostname=hostname,
app=updated_app.name)
# Users should not have the capability to include files from the
# global ignore.
Helpers.delete_path(ignore_dir)
# Defined folders/files are to move out of application.
# This is defined in the deploymentmethods.conf
# If an app is installed for the first time, all files should be included
if 'install_ignore' in updated_app.method_info and not updated_app.is_added:
Helpers.move_regexed_files(
updated_app.method_info['install_ignore'],
app_dest, ignore_dir)
# If there is a inclusion file, include files back into app.
# This is defined on a per app basis
if os.path.isfile(lookups_inclusion_location):
with open(lookups_inclusion_location, "r") as f:
lines = [l.strip() for l in f.readlines()]
lookup_inclusion_results = Helpers.move_regexed_files(
lines, ignore_dir, app_dest)
if lookup_inclusion_results['errors_found']:
Logger.warn(
"Lookup inclusion error found",
paths=lookup_inclusion_results[
'path_errors'],
hostname=hostname,
app=updated_app.name,
todo="Remove file/path from inclusion..")
# Problem with host inclusion,
# move to next host
continue
updated_app.method_info['inclusions'] = \
lookup_inclusion_results['files_moved']
# Update objects with inclusions
updated_app.copy_value_to_method_info(
'inclusions', apps_meta)
os.remove(lookups_inclusion_location)
Helpers.delete_path(ignore_dir)
if use_templating and not updated_app.method_info[
'skip_templating']:
# Can template based on vars from templated
# values, hosts vars and app vars
Helpers.template_directory(app_dest, [
self.template_values, host.to_dict,
updated_app.to_dict
])
# Should only change access and create version file if a whole app is copied
if updated_app.update_method_is_copy:
for host_path, host_dir, host_files in os.walk(
app_dest): # pylint: disable=unused-variable
for host_file in host_files:
# Splunk apps can have active binaries in multiple languages
# This is a catch all to make sure apps have all the required
# permissions.
chmod = 0755
os.chmod(os.path.join(host_path, host_file),
chmod)
if not updated_app.method_info['no_appetite_changes']:
with open(
os.path.join(
app_dest,
Helpers.get_app_version_filename()),
"w") as f:
f.write(updated_app.to_json)
AppVersioning.create_app_version(
app_dest,
updated_app.commit_log['app_abbrev_commit_id'])
apps_distro = Helpers.content_wrapper(apps_meta,
Consts.META_CURRENT,
hostname, self.track)
# Meta file used as source of truth on instance
master_meta = self.create_meta_files(tmp_hostname_meta, '',
apps_distro)
# check be used to update and test manifest changes locally
if self.args.dryrun:
Helpers.create_path(host.local_meta_file)
shutil.copy(master_meta, host.local_meta_file)
# Always want clean logs ingested
selected_apps = Helpers.select_and_update_apps(
apps_meta, Consts.META_CURRENT, False)
self.create_meta_log(tmp_hostname_meta, '', selected_apps,
Helpers.get_utc())
host.updates = Helpers.content_process(apps_meta,
Consts.META_UPDATED,
hostname, self.track, True)
# Create the meta change file
self.create_meta_files(tmp_hostname_meta, '_update',
Helpers.content_convert(host.updates))
# Clean updates file for logging
selected_apps = Helpers.select_and_update_apps(
apps_meta, Consts.META_UPDATED, True)
self.create_meta_log(tmp_hostname_meta, '_update', selected_apps,
Helpers.get_utc())
Logger.info("Changes found",
updates=Helpers.content_wrapper(
apps_meta, Consts.META_UPDATED, hostname,
self.track, True))
# Package (tar) up host tmp directories for distribution
tar = tarfile.open(
os.path.join(self.tars_folder, "%s.tar.gz" % tarname), "w:gz")
tar.add(tmp_hostname_dir, arcname=os.path.basename(self.base_name))
tar.close()
changes_found = True
if errors_found:
sys.exit(1)
self.repo_manager.set_commit_id()
return changes_found
def populate_apps_to_hosts(self):
"""Parses the manifest and adds apps to hosts
:return: None
"""
Helpers.check_file(self.manifest_path)
with open(self.manifest_path, 'rU') as csvfile:
mreader = csv.reader(csvfile, delimiter=',', quotechar='"')
first_row = True
# Go though each app
for row in mreader:
# Remove header if it exists
if first_row:
# Defines column headers in manifest
column_headers = {
col_name: -1
for col_name in Consts.DEFAULT_COLUMN_HEADER
}
# Get indexes for headers from the first row
num_columns = len(row)
for k in column_headers:
value_index = next((index
for index in range(0, num_columns)
if row[index].lower() == k), -1)
if value_index < 0:
Logger.errorout("Manifest header is missing",
header=k)
column_headers[k] = value_index
first_row = False
continue
if len(row) > 1:
row_values = Helpers.create_obj({
"commit_id":
row[column_headers['commitid']],
"app_clean":
self.deployment_manager.name_filter.sub(
"", row[column_headers['application']]),
"app":
row[column_headers['application']],
"deployment":
row[column_headers['deploymentmethod']],
"white_list":
row[column_headers['whitelist']].split(','),
"black_list":
row[column_headers['blacklist']].split(',')
})
app_folder = os.path.join(self.apps_folder, row_values.app)
if self.args.build_test_apps:
# for testing - create test folders for apps
if not os.path.exists(app_folder):
Helpers.create_path(
os.path.join(app_folder, "folder"), True)
app_test_file = "%s/%s.txt" % (
app_folder, row_values.app_clean)
with open(app_test_file, 'wb') as touch:
touch.write("")
# validate commit id
if len(row_values.commit_id
) > 0 and not Helpers.validate_commit_id(
row_values.commit_id):
Logger.critical("Invalid commit ID",
commit_id=row_values.commit_id)
# Go through each host and see
# if the app is needed for the host
for host in self.appetite_hosts:
self.add_to_host(host, row_values)
self.bootstrap_firstrun_hosts(host, row_values)
if self.args.new_host_brakes and next(
(True
for host in self.appetite_hosts if host.bootstrap), False):
self.args.num_connections = 1
if self.appetite_hosts.is_empty():
Logger.errorout("Manifest misconfiguration, "
"no apps for any hosts")