def instance_setup(instance_name):
"When Bombardier is first getting set up, this method is called."
progress_path = get_progress_path(instance_name)
status_dict = None
if os.path.isfile(progress_path):
try:
status_dict = json.loads(open(progress_path).read())
except:
msg = "Unable to load existing json from %s" % progress_path
Logger.warning(msg)
if type(status_dict) != type({}):
status_dict = {"install-progress": {}}
status_dict["client_version"] = CLIENT_VERSION
status_dict["core_version"] = CORE_VERSION
status_dict["clientVersion"] = CLIENT_VERSION
status_dict["coreVersion"] = CORE_VERSION
pkg_dir = make_path(get_spkg_path(), instance_name, "packages")
repos_dir = make_path(get_spkg_path(), "repos")
tmp_dir = make_path(get_spkg_path(), "tmp")
if not os.path.isdir(pkg_dir):
os.makedirs(pkg_dir)
if not os.path.isdir(repos_dir):
os.makedirs(repos_dir)
if not os.path.isdir(tmp_dir):
os.makedirs(tmp_dir)
open(progress_path, 'w').write(json.dumps(status_dict))
def reconcile_system(self, action):
"""
Figure out everything that needs to happen to get this machine into
compliance with its stated configuration and do it
action -- DRY_RUN or RECONCILE
"""
self.operation_status = OK
self.operation_output = []
pkns = []
dry_run = False
if action == DRY_RUN:
msg = "Reconcile dry-run starting..."
Logger.info(msg)
dry_run = True
else:
msg = "Reconcile starting..."
Logger.info(msg)
add_pkd, del_pkd, uninstall_order = self._ck_inst_stat(pkns)
# Logger.info("uninstall_order: %s" % uninstall_order)
status = self._uninstall_pkgs(del_pkd, uninstall_order, dry_run)
if status == FAIL:
msg = "Uninstallation failed. Aborting reconcile."
self.operation_output.append(msg)
self.operation_status = FAIL
return self._cleanup()
add_pkd, del_pkd, uninstall_order = self._ck_inst_stat(pkns)
status = self._install_pkgs(add_pkd, dry_run)
if status != OK:
self.operation_status = FAIL
self.operation_output.append("Finished installing")
return self._cleanup()
def execute_maint_script(self, script_name):
'''
execute a user-defined function
script_name -- name of the function to run
'''
Package.execute_maint_script(self)
script_path = "%s/%s.py" % (self.maint_dir, script_name)
start_dir = get_slash_cwd()
os.chdir(self.maint_dir)
if not os.path.isfile(script_path):
msg = "%s does not exist" % script_path
raise BadPackage, (self.name, msg)
sys.path.append(self.maint_dir )
import_string = 'import %s' % script_name
cmd = 'status = %s.execute(self.config, Logger)' % script_name
job = Job(import_string, cmd, self.config)
status = job.execute()
sys.path.remove(self.maint_dir)
os.chdir(start_dir)
if status == None:
status = OK
if type(status) != type(1):
msg = "Invalid status type (%s: '%s')"
Logger.warning(msg % (type(status), status))
status = FAIL
return status
def exit_with_return_code(value):
"Provide proper output to the CNM server"
if type(value) != type(0):
Logger.error("Invalid exit code, not an integer: %s" % value)
value = FAIL
Logger.warning("==EXIT-CODE==:%s" % value)
sys.exit(value)
def _check_bom(self, bom_pkns):
""" Check through what should be installed on the system and what
is installed on the system and determine what packages aren't
installed that should be and what packages are installed that
shouldn't be.
bom_pkns -- A list of package names that are intended to be
installed on the machine (i.e. the 'bill of
materials'
"""
should_be_installed = []
shouldnt_be_installed = []
pdat = self.progress.get_progress_data(False)
installed_pkns, broken_pkns = Progress.get_installed(pdat)
all_pkns = set(installed_pkns).union(broken_pkns)
all_plus_missing_pkns = self._examine_dependencies(all_pkns)
missing_pkns = list(all_plus_missing_pkns - all_pkns)
bom_pkns = bom_pkns + missing_pkns
dependency_errors = self._get_dependency_errors(bom_pkns, pdat)
if dependency_errors:
errmsg = "The following packages are installed as " "dependencies %s" % dependency_errors
Logger.debug(errmsg)
bom_pkns += dependency_errors
for pkn in installed_pkns:
if pkn not in bom_pkns:
shouldnt_be_installed.append(pkn)
for pkn in bom_pkns:
if pkn not in installed_pkns:
should_be_installed.append(pkn)
return should_be_installed, shouldnt_be_installed
def _uninstall_pkgs(self, del_pkd, uninstall_order, dry_run=False):
"""
Given a dictionary of packages and an uninstall order, make some
changes on this machine
del_pkd -- A dictionary of package objects that are to
be uninstalled
uninstall_order -- the order with which to uninstall packages
dry_run -- boolean flag indicating if we're serious about this or not
"""
status = OK
remove_full_pkns = []
# Logger.info("UninstallOrder: %s" % uninstall_order)
for name in uninstall_order:
if del_pkd[name].full_name:
name_str = del_pkd[name].full_name
else:
name_str = name
remove_full_pkns.append(name_str)
msg = "Packages to remove: %s" % remove_full_pkns
Logger.info(msg)
for pkn in uninstall_order:
uninstall_status = del_pkd[pkn].uninstall(dry_run)
if uninstall_status == FAIL:
return FAIL
return status
def unzip_type_4(self, pkg_path, full_name):
'''
Perform untar/ungzip operations. (NOTE: NOT RELIABLE IN Python)
pkg_path -- full path to package file, minus extension
full_name -- name of the package, including version
'''
Logger.info("Unzipping %s" % full_name)
self.untargz(pkg_path + ".spkg")
return OK
def add_dependency_error(self, dependency_name):
'''
Record the fact that the system has a dependency error.
This is used when uninstalling packages later.
dependency_name -- name of the dependency that is missing
'''
errmsg = "BOM file is incomplete: should contain %s" % dependency_name
Logger.warning(errmsg)
self.dependency_errors.append(dependency_name)
def _eval_priority(self):
'determine priority of this package'
if not self.priority:
self.priority = AVERAGE
else:
try:
self.priority = int(self.priority)
except ValueError:
Logger.warning(ermsg)
self.priority = AVERAGE
def execute_maint_script(self, script_name, arguments):
'''
execute a user-defined function
script_name -- name of the function to run
arguments -- arguments to the script to be run
'''
Package.execute_maint_script(self, script_name)
self.status = self._find_cmd(script_name, arguments=arguments)
msg = "%s result for %s : %s"
Logger.info(msg % (script_name, self.full_name, self.status))
return self.status
def process_action(action, instance_name, pkn, method_name,
arguments, package_factory):
"""
Performs a Bombardier action on this system
action -- either INIT, INSTALL, UNINSTALL, VERIFY, CONFIGURE,
EXECUTE, RECONCILE, CHECK_STATUS, FIX, or PURGE
instance_name -- the name of the machine
pkn -- the package name to operate on. Could be none for actions
that operate on the whole machine, such as RECONCILE,
CHECK_STATUS, or INIT
method_name -- Only applicable for the EXECUTE action, runs a named
method within a package.
package_factory -- provides packages we may need
"""
if action == INIT:
instance_setup(instance_name)
return OK
if action in [ UNINSTALL, VERIFY, CONFIGURE, EXECUTE, BACKUP ]:
try:
pkn = find_likely_pkn(instance_name, pkn)
except ValueError:
return FAIL
status = FAIL
try:
if action in [ FIX, PURGE ]:
status = fix_spkg(instance_name, pkn, action, package_factory)
return status
bc_obj = get_bc(instance_name, package_factory.env)
if action == CHECK_STATUS:
status_dict = bc_obj.check_system()
if type(status_dict) == type({}):
if not status_dict["Packages that are BROKEN"]:
status = OK
else:
status = FAIL
elif action in [ RECONCILE, DRY_RUN ]:
bc_obj.record_errors = True
status = bc_obj.reconcile_system(action)
else: # INSTALL, UNINSTALL, VERIFY, BACKUP, CONFIGURE, EXECUTE, RESTORE
bc_obj.record_errors = False
status = bc_obj.use_pkg(pkn, action, method_name, arguments)
except:
err = StringIO.StringIO()
traceback.print_exc(file = err)
err.seek(0)
data = err.read()
ermsg = ''
for line in data.split('\n'):
ermsg += "\n||>>>%s" % line
Logger.error(ermsg)
return FAIL
return status
def initialize(self):
'''Evaluate the package meta-data for consistency
and initialize parameters. Verify that data on the disk for
this package is consistent.
'''
try:
self.meta_data = self.repository.get_meta_data(self.name)
except BadPackage, bpe:
Logger.error("initialize INVALIDATING")
self._invalidate(bpe)
return
def verify(self, dry_run=False):
'''
Perform a verification action on this package
dry_run -- whether to actually perform the action
'''
self._download()
message = "Verifying package %s" % self.full_name
Logger.info(message)
self.status = self._find_cmd(VERIFY, dry_run=dry_run)
if self.action != INSTALL:
self._write_progress()
Logger.info("Verify result for %s : %s" % (self.full_name, self.status))
return self.status
def execute_maint_script(self, script_name):
'''
execute a user-defined function
script_name -- name of the function to run
'''
self._download()
# remove old history
output_path = make_path(get_spkg_path(), self.instance_name,
"output", "%s-output.yml" % script_name)
if os.path.isfile(output_path):
os.unlink(output_path)
message = "Executing (%s) inside package (%s)"
Logger.info(message % (script_name, self.full_name))
def find_likely_pkn(instance_name, pkn):
"""Sometimes an ambiguous package name is requested.
This attempts to help a guy out."""
progress_file = get_progress_path(instance_name)
try:
status_yml = json.loads(open(progress_file).read())
except ValueError:
Logger.error("Progress data in %s is not parsable." % progress_file)
raise
pkns = []
status_packages = status_yml['install-progress']
for name in status_packages:
if status_packages[name]['INSTALLED'] in [ 'NA', 'BROKEN' ]:
continue
if pkn.lower() in name.lower():
pkns.append(name)
if len(pkns) > 1:
msg = 'Ambiguous package name: %s could be any of %s' % (pkn, str(pkns))
Logger.error( msg )
exit_with_return_code(FAIL)
if len(pkns) == 0:
Logger.error( 'Package not found: %s' %pkn )
exit_with_return_code(FAIL)
else:
pkn = '-'.join(pkns[0].split('-')[:-1])
Logger.info( 'Using %s' %pkn)
return pkn
def is_running(self):
'ability to check if the job is finished'
if self.job_thread:
if self.job_thread.isAlive():
return True
if type(self.job_thread.cmd_status) == type(0) \
or type(self.job_thread.cmd_status) == type('str'):
Logger.debug("-- status: %s" % (self.job_thread.cmd_status))
self.job_status = self.job_thread.cmd_status
else:
msg = "Invalid return status (type: %s)"
Logger.error(msg % (type(self.job_thread.cmd_status)))
self.job_status = FAIL
return False
def _gather_dependencies(self):
'determine what packages must be installed before this package'
self.dependencies = self.meta_data.data.get("dependencies")
if type(self.dependencies) == type(None):
self.dependencies = []
elif type(self.dependencies) == type('string'):
self.dependencies = [self.dependencies]
elif type(self.dependencies) == type([]):
pass
else:
errmsg = "Unsupported dependencies data for %s. "\
"Must be a list. [%s] instead."
Logger.warning(errmsg % (self.name, self.dependencies))
self.dependencies = []
def _get_del_pkd(self, del_pkns):
"""
create a dictionary of package objects based on a list of names
that we know we're supposed to remove
del_pkns -- the names of packages that we should be removing
"""
pkd = {}
for pkn in del_pkns:
try:
pkg = self._get_new_pkg(pkn)
pkg.action = UNINSTALL
pkd[pkn] = pkg
except Exceptions.BadPackage, err:
errmsg = "Skipping Bad package: %s" % err
Logger.warning(errmsg)
def _filter_bad_pkgs(self):
"""
Go through my chain of packages, and if there are any 'bad' packages
in the list of dependencies, cut off the package chain at that point
"""
index = 0
for index in range(0, len(self.chain)):
pkn = self.chain[index]
if pkn in self.broken_pkns:
index -= 1
msg = "Omitting packages %s due to bad package"
msg = msg % self.chain[index:]
Logger.warning(msg)
break
self.chain = self.chain[: index + 1]
def _get_add_pkd(self, add_pkns):
"""
create a pkd for package names that should be installed
add_pkns -- the names of packages that we should be installing
"""
pkd = {}
for pkn in add_pkns:
try:
pkg = self._get_new_pkg(pkn)
self._check_configuration(pkg)
pkd[pkn] = pkg
except Exceptions.BadPackage, err:
errmsg = "Skipping bad package %s: %s" % (err.pkn, err.errmsg)
self.operation_output.append(errmsg)
Logger.warning(errmsg)
def _install(self, future_pkns, dry_run=False):
'''
perform an installation operation
future_pkns -- package names to be installed after this one
dry_run -- whether or not to actuall perform the installation
'''
dry_run_string = ""
if dry_run:
dry_run_string = " --DRY_RUN-- "
self._download()
message = "Beginning installation of (%s)%s"
Logger.info(message % (self.full_name, dry_run_string))
self.status = self._find_cmd(INSTALL, future_pkns=future_pkns, dry_run=dry_run)
msg = "Install result for %s%s : %s"
Logger.info(msg % (self.full_name, dry_run_string, self.status))
return self.status
def _find_cmd(self, action, arguments=[], future_pkns=[], dry_run=False):
'''
Perform the action on the system, importing modules from the package
and running the appropriate method on the class within.
action -- INSTALL, UNINSTALL, CONFIGURE, VERIFY
future_pkns -- future package names. Some packages want to know
about the packages that will come after them
dry_run -- boolean flag to see if we're really going to do this
'''
ret_val = None
if type(action) == type(1):
action = ACTION_REVERSE_LOOKUP[action]
cwd = get_slash_cwd()
obj, rand_string = self._get_object(future_pkns)
try:
if not hasattr(obj, action):
msg = "Class %s does not have a %s method."
raise BadPackage(self.name, msg % (self.class_name, action))
if not dry_run:
if arguments:
if ACTION_LOOKUP.get(action) == RESTORE:
if len(arguments) != 1:
Logger.error("Incorrect number of arguments passed to restore")
return FAIL
restore_path = make_path(get_spkg_path(), "archive",
self.name, str(arguments[0]))
if not os.path.isdir(restore_path):
msg = "Cannot execute restore: archive data does not "\
"exist in %s" % (restore_path)
Logger.error(msg)
return FAIL
self._prepare_restore(obj, restore_path)
exec("ret_val = obj.%s('%s')" % (action, restore_path))
else:
exec("ret_val = obj.%s()" % (action))
else:
ret_val = OK
self._cleanup(obj)
del rand_string
except SystemExit, err:
if err.code:
ret_val = err.code
else:
ret_val = OK
del rand_string
def use_pkg(self, pkn, action, script_name, arguments):
"""
Main entry point to the class. Performs an action using a package
pkn -- name of the package to use
action -- STATUS, INSTALL, UNINSTALL, CONFIGURE, VERIFY, or EXEC
script_name -- the name of a method to run within the package
arguments -- arguments to the executable, typically a restore target
"""
try:
pkg = self._get_new_pkg(pkn)
if pkg.status == FAIL:
self.operation_status = FAIL
return self._cleanup()
if action == INSTALL:
pdat = self.progress.get_progress_data(False)
installed_pkns, broken_pkns = Progress.get_installed(pdat)
if pkn in [installed_pkns + broken_pkns]:
Logger.error("Package %s cannot be installed." % pkn)
self.operation_status = FAIL
return FAIL
add_pkd = {pkn: pkg}
status = self._install_pkgs(add_pkd)
if action == UNINSTALL:
pdat = self.progress.get_progress_data(False)
installed_pkns, broken_pkns = Progress.get_installed(pdat)
bom_pkns = installed_pkns
if pkn in bom_pkns:
bom_pkns.remove(pkn)
self.config.set_bom_pkgs(bom_pkns)
add_pkd, del_pkd, uninstall_order = self._ck_inst_stat([])
status = self._uninstall_pkgs(del_pkd, uninstall_order)
if action == VERIFY:
status = pkg.verify()
if action == CONFIGURE:
self._check_configuration(pkg)
status = pkg.configure()
hash_path = make_path(pkg.get_path(), HASH_FILE)
msg = "Writing configuration fingerprint to %s" % hash_path
Logger.info(msg)
self.config.save_hash(hash_path)
if action in [EXECUTE, BACKUP, RESTORE]:
status = pkg.execute_maint_script(script_name, arguments)
if status == FAIL:
self.operation_status = FAIL
msg = "Finished %s for %s."
msg = msg % (ACTION_REVERSE_LOOKUP[action], pkn)
self.operation_output.append(msg)
status = self._cleanup()
return self._cleanup()
except Exceptions.BadPackage, err:
errmsg = "Cannot perform action %s on package %s: %s"
errmsg = errmsg % (ACTION_REVERSE_LOOKUP[action], err.pkn, err.errmsg)
Logger.warning(errmsg)
Logger.info("RETURN FAIL 1")
return FAIL
def _check_configuration(self, pkg):
"""
Check to see if we have sufficient configuration data to mess around
with a given package
pkg -- a package object that we need to do something with
"""
if pkg.get_configuration():
required_configs = pkg.get_configuration()
diff = diff_dicts(required_configs, self.config.data)
if diff != {}:
self.operation_status = FAIL
errmsg = "This machine does not have sufficient " "configuration data to install %s " % pkg.name
diff_txt = ""
for key in diff:
diff_txt += "key: %s; value: %s" % (key, diff[key])
Logger.warning(errmsg)
Logger.warning(diff_txt)
raise Exceptions.BadPackage(pkg.name, "Bad Config")
def _ck_inst_stat(self, bom_pkns):
"""
Given a list of packages that are supposed to be on this machine, we
will look at the installation status and (1) make a decision on what
packages should be removed (and in what order) and (2) decide which
packages should be added.
bom_pkns -- A list of package names that are intended to be
installed on the machine (i.e. the 'bill of
materials'
"""
if bom_pkns == []:
bom_pkns = self.config.get_bom_pkns()
if bom_pkns == []:
Logger.warning("Empty Bill of Materials")
add_pkns, del_pkns = self._check_bom(bom_pkns)
add_pkd = self._get_add_pkd(add_pkns)
del_pkd, uninstall_order = self._get_pkd_to_remove(del_pkns)
return add_pkd, del_pkd, uninstall_order
def uninstall(self, dry_run=False):
'''
Perform an uninstall action on this package
dry_run -- whether to actually perform the action
'''
self.action = UNINSTALL
self._download()
if self.status != OK:
return self.status
dry_run_string = ""
if dry_run:
dry_run_string = " --DRY_RUN-- "
msg = "Uninstalling package %s%s" % (self.name, dry_run_string)
Logger.info(msg)
self.status = self._find_cmd(UNINSTALL, dry_run=dry_run)
if not dry_run:
self._write_progress()
msg = "Uninstall result for %s%s : %s"
Logger.info(msg % (self.full_name, dry_run_string, self.status))
return self.status
def check_system(self):
"""
After a system has had all of its packages installed and it's in
stable-state, this is a method we can use to verify that everything
is still kosher
"""
Logger.info("System-check starting...")
bom_pkns = self.config.get_bom_pkns()
pdat = self.progress.get_progress_data()
full_pdat = self.progress.get_progress_data(False)
full_installed_pkns, _full_bpkns = Progress.get_installed(full_pdat)
msg = "Packages that are installed: %s"
Logger.info(msg % " ".join(full_installed_pkns))
installed_pkns, broken_pkns = Progress.get_installed(pdat)
should_be_installed, shouldnt_be_installed = self._check_bom(bom_pkns)
# check the configuration for each installed package
pkg_info = {
"Packages installed properly": installed_pkns,
"Packages to be RECONFIGURED": {},
"Packages to be INSTALLED": should_be_installed,
"Packages to be REMOVED": shouldnt_be_installed,
"Packages that are BROKEN": broken_pkns,
}
for pkn in shouldnt_be_installed:
pkg_info["Packages installed properly"].remove(pkn)
for pkn in installed_pkns:
differences = self._check_configuration_hash(pkn)
if differences:
if pkn in pkg_info["Packages installed properly"]:
pkg_info["Packages to be RECONFIGURED"][pkn] = differences
pkg_info["Packages installed properly"].remove(pkn)
for key in pkg_info:
Logger.info("==OUTPUT==:%s: %s" % (key, pkg_info[key]))
return pkg_info
def _create_pkg_chains(self, pkd):
"""
Create package chains based on current set of packages
pkd -- dictionary of package objects
"""
chains = []
pkg_data = self.progress.get_progress_data(False)
installed_pkns, broken_pkns = Progress.get_installed(pkg_data)
for pkn in pkd.keys():
if pkn in broken_pkns:
Logger.warning("Skipping broken package %s" % pkn)
continue
if pkn not in installed_pkns:
chain_priority = pkd[pkn].priority
else:
chain_priority = 0
try:
new_chain = PackageChain(
chain_priority,
pkn,
pkd,
installed_pkns,
broken_pkns,
self.repository,
self.config,
self.instance_name,
self.record_errors,
)
except Exceptions.BadPackage, err:
errmsg = "Package %s will not be installed because it is " "dependent upon one or more broken packages"
Logger.warning(errmsg % pkn)
Logger.warning(str(err))
continue
chains.append([new_chain.priority, new_chain.chain])
def _sort_uninstalled_pkgs(self, uninstall_order):
"""Make sure packages get uninstalled in the right order, paying
attention to dependencies.
uninstall_order -- a list of package names the order of which indicates
which should be uninstalled first, second...
"""
dependency_dict = {}
if len(uninstall_order) > 1:
Logger.info("Determining uninstallation order...")
for pkn in uninstall_order:
dependency_dict[pkn] = []
for pkn in uninstall_order:
pkg = self._get_new_pkg(pkn)
for other_pkn in uninstall_order:
if other_pkn in pkg.dependencies:
dependency_dict[other_pkn].append(pkn)
else:
return uninstall_order
new_proper_order = copy.deepcopy(uninstall_order)
swapped = True
while swapped:
proper_order = copy.deepcopy(new_proper_order)
swapped = False
for pkn in proper_order:
dependency_list = dependency_dict[pkn]
for dependency in dependency_list:
index1 = proper_order.index(dependency)
index2 = proper_order.index(pkn)
if index1 > index2:
new_proper_order = swap(proper_order, index1, index2)
swapped = True
break
if swapped:
break
Logger.info("Uninstallation Order: %s" % new_proper_order)
return new_proper_order
def _install_pkgs(self, add_pkd, dry_run=False):
"""
Provided a dictionary of packages, we will install those that need to
be installed.
add_pkd -- a dictionary of packages that need to be added
dry_run -- whether or not to just 'fake it'
"""
status = OK
making_progress = True
pkns_left = ["initialize"]
while making_progress and pkns_left:
making_progress = False
install_order = self._find_install_order(add_pkd)
pkns_left = list(install_order)
for pkn in install_order:
msg = "Packages remaining to install (in order):"
Logger.info(msg)
for tpkn in pkns_left:
Logger.info(" + %s" % tpkn)
pkns_left.remove(pkn)
pkg = add_pkd[pkn]
erstr = "Currently installing package priority %s [%s]"
Logger.info(erstr % (pkg.get_priority(), pkn))
pkg_status = pkg.install_and_verify(pkns_left)
if not dry_run:
hash_path = make_path(pkg.get_path(), HASH_FILE)
self.config.save_hash(hash_path)
if pkg_status == FAIL:
status = FAIL
erstr = "Package installation failure -- re-calculating" " package installation order"
Logger.error(erstr)
break
else:
making_progress = True
if status != OK:
msg = "There are packages that are broken, and we have done all" " we can do. ; ;"
Logger.error(msg)
return status
