def process_hosts(self):
if not self.args.tmp_folder:
Logger.errorout("tmp folder must be defined")
# Deleting the tmp folder to keep installs clean
Helpers.delete_path(self.tmp_folder)
try:
if self.args.template_files:
template_paths = self.args.template_files
# Incase one long string is entered
if isinstance(self.args.template_files, basestring):
template_paths = self.args.template_files.split(' ')
self.template_values = Helpers.load_templating(template_paths)
except Exception as exception:
Logger.errorout("No templating problem: %s" % exception.message)
if self.args.template_json:
try:
self.template_values.update(
json.loads(self.args.template_json.replace('\\"', '"')))
except Exception as err:
Logger.errorout("Error parsing --template-json",
error=err.message)
if self.args.template_filtering:
self.template_values = Helpers.filter_object(
self.template_values, self.args.template_filtering)
ConnManager.set_globals(self.args.ssh_user, self.args.ssh_keyfile,
self.args.ssh_port, self.args.app_folder,
self.args.app_binary, self.args.dryrun)
# Load any files reference to appetite scripts folder before this
# Working directories change with repo management
repo_status = self.repo_manager.pull_repo(self.args.clean_repo)
if not self.args.dryrun and repo_status < 0:
Logger.errorout('Repo Error, Look at logs for details')
repo_check_status = self.repo_manager.check_for_update()
Logger.add_track_info(self.repo_manager.track)
triggered = repo_check_status['triggered'] or repo_status == 1
Logger.info('Repo pull',
output=repo_check_status['output'],
triggered=triggered)
if not self.args.dryrun and not self.args.skip_repo_sync and not triggered:
Logger.info('No repo update found', complete=False)
self.print_track_info(False)
sys.exit(0)
self.repo_manager.set_commit_id()
# Load in deploymentmethods.conf
self.deployment_manager = DeploymentMethodsManager(
self.repo_name, "", self.scratch_location,
self.args.deployment_methods_file)
# Generate hosts
if self.args.hosts:
# Incase one long string is entered
if len(self.args.hosts) == 1:
self.args.hosts = self.args.hosts[0].split(' ')
for host in self.args.hosts:
split_hostname = host.strip("'\"").split(':')
clean_hostname = split_hostname[0].split('.')[0].strip("'\"")
# With user name, the ssh hostname can be defined. This allows
# for IP addresses to be defined incase there is no DNS.
host_data = Helpers.pull_class_from_host(
self.name_formatting, clean_hostname, self.host_classes)
if host_data:
# Can use a specified hostname/IP.
# Default is the given hostname
ssh_host = split_hostname[len(split_hostname) - 1]
self.appetite_hosts.add_host(self, clean_hostname,
host_data, ssh_host)
else:
# Create hosts based on classes
for host_class in self.host_classes:
self.appetite_hosts.add_host(
self,
Helpers.build_hostname(
self.name_formatting, # pylint: disable=no-value-for-parameter
host_class,
1))
if self.appetite_hosts.is_empty():
Logger.errorout("No hosts found after filtering")
if self.args.clean_metas:
Helpers.delete_path(self.meta_folder)
# Only update if a manifest file is not found
self.update_manifests(check_if_exists=True)
Logger.info("appetite started",
use_templating=self.args.templating,
firstrun=self.args.firstrun)
self.populate_apps_to_hosts()
self.ssh_app_commands = ConnManager.SshAppCommands(
self.app_commands_file, self.template_values)
changes_found = self.create_host_directories_and_tar()
if changes_found:
Logger.info("Start host updates")
self.update_hosts()
Logger.info("End host updates")
self.print_track_info(changes_found)
Logger.info("Appetite complete", complete=True, changes=changes_found)
def populate_apps_to_hosts(self):
"""Parses the manifest and adds apps to hosts
:return: None
"""
Helpers.check_file(self.manifest_path)
with open(self.manifest_path, 'rU') as csvfile:
mreader = csv.reader(csvfile, delimiter=',', quotechar='"')
first_row = True
# Go though each app
for row in mreader:
# Remove header if it exists
if first_row:
# Defines column headers in manifest
column_headers = {
col_name: -1
for col_name in Consts.DEFAULT_COLUMN_HEADER
}
# Get indexes for headers from the first row
num_columns = len(row)
for k in column_headers:
value_index = next((index
for index in range(0, num_columns)
if row[index].lower() == k), -1)
if value_index < 0:
Logger.errorout("Manifest header is missing",
header=k)
column_headers[k] = value_index
first_row = False
continue
if len(row) > 1:
row_values = Helpers.create_obj({
"commit_id":
row[column_headers['commitid']],
"app_clean":
self.deployment_manager.name_filter.sub(
"", row[column_headers['application']]),
"app":
row[column_headers['application']],
"deployment":
row[column_headers['deploymentmethod']],
"white_list":
row[column_headers['whitelist']].split(','),
"black_list":
row[column_headers['blacklist']].split(',')
})
app_folder = os.path.join(self.apps_folder, row_values.app)
if self.args.build_test_apps:
# for testing - create test folders for apps
if not os.path.exists(app_folder):
Helpers.create_path(
os.path.join(app_folder, "folder"), True)
app_test_file = "%s/%s.txt" % (
app_folder, row_values.app_clean)
with open(app_test_file, 'wb') as touch:
touch.write("")
# validate commit id
if len(row_values.commit_id
) > 0 and not Helpers.validate_commit_id(
row_values.commit_id):
Logger.critical("Invalid commit ID",
commit_id=row_values.commit_id)
# Go through each host and see
# if the app is needed for the host
for host in self.appetite_hosts:
self.add_to_host(host, row_values)
self.bootstrap_firstrun_hosts(host, row_values)
if self.args.new_host_brakes and next(
(True
for host in self.appetite_hosts if host.bootstrap), False):
self.args.num_connections = 1
if self.appetite_hosts.is_empty():
Logger.errorout("Manifest misconfiguration, "
"no apps for any hosts")
def __init__(self, is_already_running):
self.args = parse_args()
# default path for configs
self.app_config_dir = os.path.abspath("../config")
# Update args from config file
if self.args.config_file:
abs_config_file = os.path.abspath(
os.path.dirname(self.args.config_file))
self.args.__dict__.update(
AppetiteArgs.load_args(self.args.config_file))
self.app_config_dir = abs_config_file
self.app_commands_file = os.path.join(self.app_config_dir,
"commands.conf")
if self.args.command_conf:
self.app_commands_file = os.path.join(
os.path.abspath(os.path.expandvars(self.args.command_conf)))
AppetiteArgs.args_check(self.args)
# Set up logging after args are set
Logger.setup_logging('appetite_%s' % self.args.refname,
self.args.refname, True,
self.args.disable_logging, self.args.silent,
self.args.logging_path)
if is_already_running:
Logger.info("Appetite is already processing")
self.print_track_info(False, Helpers.get_track())
return
# Get host classes for filtering
self.host_classes = self.args.host_classes.split(" ") \
if isinstance(self.args.host_classes, basestring) \
else self.args.host_classes
# Reverse sorting needed for name filtering
self.host_classes.sort(reverse=True)
# Get boot ordering
self.boot_ordering = Helpers.get_enchanced_boot_order(
self.args.boot_order, self.host_classes)
self.repo_name = self.args.repo_url.split('/')[-1].split('.')[0]
self.scratch_location = os.path.join(
os.path.abspath(os.path.expandvars(self.args.scratch_dir)),
self.args.refname)
self.repo_path = os.path.join(self.scratch_location, self.repo_name)
self.apps_folder = os.path.join(self.repo_path, self.args.apps_folder)
self.manifest_path = os.path.join(self.repo_path,
Consts.CONFIG_PATH_NAME,
self.args.apps_manifest)
self.tmp_folder = os.path.join(self.scratch_location,
self.args.tmp_folder)
self.meta_folder = os.path.join(self.scratch_location, 'meta')
self.tars_folder = os.path.join(self.tmp_folder, 'tars')
self.hosts_folder = os.path.join(self.tmp_folder, 'hosts')
self.remote_apps_path = os.path.normpath(self.args.app_folder)
self.base_location, self.base_name = os.path.split(
self.remote_apps_path)
self.name_formatting = self.args.name_formatting.strip('"\'')
self.meta_name = "%s%s" % (Consts.APPS_METADATE_FILENAME,
self.args.refname)
self.meta_remote_folder = os.path.join(self.remote_apps_path,
Consts.META_DIR)
self.meta_remote_logs_folder = os.path.join(
self.meta_remote_folder, Consts.HOST_LOGS_FOLDER_NAME)
self.meta_remote_file = "%s.json" % os.path.join(
self.meta_remote_folder, self.meta_name)
if self.args.clean:
Helpers.delete_path(self.scratch_location)
self.repo_manager = RepoManager(self.repo_name, self.args.repo_url,
self.args.repo_branch, "",
self.scratch_location,
self.args.apps_manifest,
self.args.dryrun)
Logger.debug_on(self.args.debug)
if not self.args.tmp_folder:
Logger.errorout("tmp folder must be defined")
# Deleting the tmp folder to keep installs clean
Helpers.delete_path(self.tmp_folder)
self.template_values = {}
try:
if self.args.template_files:
template_paths = self.args.template_files
# Incase one long string is entered
if isinstance(self.args.template_files, basestring):
template_paths = self.args.template_files.split(' ')
self.template_values = Helpers.load_templating(template_paths)
except Exception as exception:
Logger.errorout("No templating problem: %s" % exception.message)
if self.args.template_json:
try:
self.template_values.update(
json.loads(self.args.template_json.replace('\\"', '"')))
except Exception as err:
Logger.errorout("Error parsing --template-json",
error=err.message)
if self.args.template_filtering:
self.template_values = Helpers.filter_object(
self.template_values, self.args.template_filtering)
ConnManager.set_globals(self.args.ssh_user, self.args.ssh_keyfile,
self.args.ssh_port, self.args.app_folder,
self.args.app_binary, self.args.dryrun)
self.ssh_app_commands = ConnManager.SshAppCommands(
self.app_commands_file, self.template_values)
# Load any files reference to appetite scripts folder before this
# Working directories change with repo management
repo_status = self.repo_manager.pull_repo(self.args.clean_repo)
if not self.args.dryrun and repo_status < 0:
Logger.errorout('Repo Error, Look at logs for details')
repo_check_status = self.repo_manager.check_for_update()
Logger.add_track_info(self.repo_manager.track)
triggered = repo_check_status['triggered'] or repo_status == 1
Logger.info('Repo pull',
output=repo_check_status['output'],
triggered=triggered)
if not self.args.dryrun and not self.args.skip_repo_sync and not triggered:
Logger.info('No repo update found', complete=False)
self.print_track_info(False)
sys.exit(0)
self.repo_manager.set_commit_id()
# Load in deploymentmethods.conf
self.deployment_manager = DeploymentMethodsManager(
self.repo_name, "", self.scratch_location)
# Generate hosts
if self.args.hosts:
# Incase one long string is entered
if len(self.args.hosts) == 1:
self.args.hosts = self.args.hosts[0].split(' ')
for host in self.args.hosts:
split_hostname = host.strip("'\"").split(':')
clean_hostname = split_hostname[0].split('.')[0].strip("'\"")
# With user name, the ssh hostname can be defined. This allows
# for IP addresses to be defined incase there is no DNS.
host_data = Helpers.pull_class_from_host(
self.name_formatting, clean_hostname, self.host_classes)
if host_data:
# Can use a specified hostname/IP.
# Default is the given hostname
ssh_host = split_hostname[len(split_hostname) - 1]
self.appetite_hosts.add_host(self, clean_hostname,
host_data, ssh_host)
else:
# Create hosts based on classes
for host_class in self.host_classes:
self.appetite_hosts.add_host(
self,
Helpers.build_hostname(
self.name_formatting, # pylint: disable=no-value-for-parameter
host_class,
1))
if self.appetite_hosts.is_empty():
Logger.errorout("No hosts found after filtering")
if self.args.clean_metas:
Helpers.delete_path(self.meta_folder)
# Only update if a manifest file is not found
self.update_manifests(check_if_exists=True)
Logger.info("appetite started",
use_templating=self.args.templating,
firstrun=self.args.firstrun)
self.populate_apps_to_hosts()
changes_found = self.create_host_directories_and_tar()
if changes_found:
Logger.info("Start host updates")
self.update_hosts()
Logger.info("End host updates")
self.print_track_info(changes_found)
Logger.info("Appetite complete", complete=True, changes=changes_found)