def get_and_print_inplace_risk(verbose, inplace_risk):
"""
The function browse throw the list and find first
inplace_risk and return corresponding status.
If verbose mode is used then it prints out
all inplace risks higher then SLIGHT.
"""
risks = {
'NONE:': 0,
'SLIGHT:': 1,
'MEDIUM:': 2,
'HIGH:': 3,
'EXTREME:': 4,
}
return_value = -1
for key, val in sorted(six.iteritems(risks),
key=itemgetter(1),
reverse=False):
matched = [x for x in inplace_risk if key in x]
if matched:
# if matched and return_value the remember her
if return_value < val:
return_value = val
# If verbose mode is used and value is bigger then 0 then prints out
if int(verbose) > 1:
log_message('\n'.join(matched), print_output=verbose)
elif int(verbose) == 1 and val > 0:
log_message('\n'.join(matched), print_output=verbose)
return return_value
def get_partition_layout(self, lsblk, vgs, lvdisplay):
"""
Returns dictionary with partition and realname and size
:param filename: filename with partition_layout in /root/preupgrade/kickstart directory
:return: dictionary with layout
"""
lsblk_filename = os.path.join(settings.KS_DIR, lsblk)
try:
layout = get_file_content(lsblk_filename,
'rb',
method=True,
decode_flag=False)
except IOError:
log_message(
"File %s was not generated by a content. Kickstart does not contain partitioning layout"
% lsblk_filename)
self.part_layout = None
return None
vg_info = []
lv_info = []
if vgs is not None:
vg_info = KickstartGenerator.get_volume_info(vgs, 0, 5)
if lvdisplay is not None:
lv_info = KickstartGenerator.get_volume_info(lvdisplay, 0, 1)
pg = PartitionGenerator(layout, vg_info, lv_info)
pg.generate_partitioning()
self.part_layout.extend(pg.get_partitioning())
def get_and_print_inplace_risk(verbose, inplace_risk):
"""
The function browse throw the list and find first
inplace_risk and return corresponding status.
If verbose mode is used then it prints out
all inplace risks higher then SLIGHT.
"""
risks = {
'NONE:': 0,
'SLIGHT:': 1,
'MEDIUM:': 2,
'HIGH:': 3,
'EXTREME:': 4,
}
return_value = -1
for key, val in sorted(six.iteritems(risks), key=itemgetter(1), reverse=False):
matched = [x for x in inplace_risk if key in x]
if matched:
# if matched and return_value the remember her
if return_value < val:
return_value = val
# If verbose mode is used and value is bigger then 0 then prints out
if int(verbose) > 1:
log_message('\n'.join(matched), print_output=verbose)
elif int(verbose) == 1 and val > 0:
log_message('\n'.join(matched), print_output=verbose)
return return_value
def get_all_postupgrade_files(dummy_verbose, dir_name):
"""Function gets all postupgrade files from dir_name"""
postupg_scripts = []
for root, dummy_sub_dirs, files in os.walk(dir_name):
# find all files in this directory
postupg_scripts.extend([os.path.join(root, x) for x in files])
if not postupg_scripts:
log_message("No postupgrade scripts available")
return postupg_scripts
def get_all_postupgrade_files(dummy_verbose, dir_name):
"""Function gets all postupgrade files from dir_name"""
postupg_scripts = []
for root, dummy_sub_dirs, files in os.walk(dir_name):
# find all files in this directory
postupg_scripts.extend([os.path.join(root, x) for x in files])
if not postupg_scripts:
log_message("No postupgrade scripts available")
return postupg_scripts
def check_or_create_temp_dir(temp_dir, mode=None):
"""Check if provided temp dir is valid."""
if os.path.isdir(temp_dir):
if not os.access(temp_dir, os.W_OK):
log_message("Directory %s is not writable." % temp_dir, level=logging.ERROR)
raise IOError("Directory %s is not writable." % temp_dir)
else:
os.makedirs(temp_dir)
if mode:
os.chmod(temp_dir, mode)
return temp_dir
def get_volume_info(filename, first_index, second_index):
try:
volume_list = get_file_content(os.path.join(settings.KS_DIR, filename), 'rb', method=True, decode_flag=False)
except IOError:
log_message("File %s is missing. Partitioning layout has not to be complete." % filename, level=logging.WARNING)
return None
volume_info = {}
for line in volume_list:
fields = line.strip().split(':')
volume_info[fields[first_index]] = fields[second_index]
return volume_info
def copy_kickstart_templates():
# Copy kickstart files (/usr/share/preupgrade/kickstart) for kickstart generation
for file_name in settings.KS_TEMPLATES:
target_name = os.path.join(settings.KS_DIR, file_name)
source_name = os.path.join(settings.source_dir, 'kickstart', file_name)
if not os.path.exists(target_name) and os.path.exists(source_name):
try:
shutil.copy(source_name, target_name)
except IOError:
log_message("Copying %s to %s failed" % (source_name, target_name))
pass
def list_contents(source_dir):
"""Function returns a list of installed contents"""
content_dict = {}
is_dir = lambda x: os.path.isdir(os.path.join(source_dir, x))
dirs = os.listdir(source_dir)
for dir_name in filter(is_dir, dirs):
full_dir_name = os.path.join(source_dir, dir_name, settings.content_file)
if os.path.exists(full_dir_name):
log_message('%s' % dir_name, print_output=False)
content_dict[dir_name] = full_dir_name
return content_dict
def check_or_create_temp_dir(temp_dir, mode=None):
"""Check if provided temp dir is valid."""
if os.path.isdir(temp_dir):
if not os.access(temp_dir, os.W_OK):
log_message("Directory %s is not writable." % temp_dir,
level=logging.ERROR)
raise IOError("Directory %s is not writable." % temp_dir)
else:
os.makedirs(temp_dir)
if mode:
os.chmod(temp_dir, mode)
return temp_dir
def copy_kickstart_templates():
# Copy kickstart files (/usr/share/preupgrade/kickstart) for kickstart generation
for file_name in settings.KS_TEMPLATES:
target_name = os.path.join(settings.KS_DIR, file_name)
source_name = os.path.join(settings.source_dir, 'kickstart',
file_name)
if not os.path.exists(target_name) and os.path.exists(source_name):
try:
shutil.copy(source_name, target_name)
except IOError:
log_message("Copying %s to %s failed" %
(source_name, target_name))
pass
def postupgrade_scripts(verbose, dirname):
"""
The function runs postupgrade directory
If dir does not exists the report and return
"""
if not os.path.exists(dirname):
log_message('There is no any %s directory' % settings.postupgrade_dir,
level=logging.WARNING)
return
postupg_scripts = get_all_postupgrade_files(verbose, dirname)
if not postupg_scripts:
return
#max_length = max(list([len(x) for x in postupg_scripts]))
log_message('Running postupgrade scripts:')
for scr in sorted(postupg_scripts):
interpreter = get_interpreter(scr, verbose=verbose)
if interpreter is None:
continue
log_message('Executing script %s' % scr)
cmd = "{0} {1}".format(interpreter, scr)
run_subprocess(cmd, print_output=False, shell=True)
log_message("Executing script %s ...done" % scr)
def postupgrade_scripts(verbose, dirname):
"""
The function runs postupgrade directory
If dir does not exists the report and return
"""
if not os.path.exists(dirname):
log_message('There is no any %s directory' % settings.postupgrade_dir,
level=logging.WARNING)
return
postupg_scripts = get_all_postupgrade_files(verbose, dirname)
if not postupg_scripts:
return
#max_length = max(list([len(x) for x in postupg_scripts]))
log_message('Running postupgrade scripts:')
for scr in sorted(postupg_scripts):
interpreter = get_interpreter(scr, verbose=verbose)
if interpreter is None:
continue
log_message('Executing script %s' % scr)
cmd = "{0} {1}".format(interpreter, scr)
run_subprocess(cmd, print_output=False, shell=True)
log_message("Executing script %s ...done" % scr)
def prepare_scan_system(self):
"""Function cleans previous scan and creates relevant directories"""
# First of all we need to delete the older one assessment
self.clean_scan()
self.prepare_scan_directories()
scenario = self.get_scenario()
if scenario is None:
log_message('Invalid scenario: %s' % self.conf.contents)
sys.exit(3)
scenario_path = os.path.join(self.conf.source_dir, scenario)
if not os.path.isdir(scenario_path):
log_message('Invalid scenario: %s' % scenario,
level=logging.ERROR)
sys.exit(3)
def get_interpreter(filename, verbose=False):
"""
The function returns interpreter
Checks extension of script and first line of script
"""
script_types = {"/bin/bash": ".sh", "/usr/bin/python": ".py", "/usr/bin/perl": ".pl"}
inter = list(k for k, v in six.iteritems(script_types) if filename.endswith(v))
content = get_file_content(filename, "rb")
if inter and content.startswith("#!" + inter[0]):
return inter
else:
if verbose:
log_message("Problem with getting interpreter", level=logging.ERROR)
return None
def generate(self):
if not self.collect_data():
log_message("Important data are missing for kickstart generation.", level=logging.ERROR)
return None
packages = self.output_packages()
if packages:
self.ks.handler.packages.add(packages)
self.ks.handler.packages.handleMissing = KS_MISSING_IGNORE
self.update_repositories(self.repos)
self.update_users(self.filter_kickstart_users())
self.get_partition_layout('lsblk_list', 'vgs_list', 'lvdisplay')
self.update_partitioning()
self.embed_script(self.latest_tarball)
self.delete_obsolete_issues()
self.save_kickstart()
return True
def tarball_result_dir(result_file, dirname, quiet, direction=True):
"""
pack results to tarball
direction is used as a flag for packing or extracting
For packing True
For unpacking False
"""
current_dir = os.getcwd()
tar_binary = "/bin/tar"
current_time = get_current_time()
cmd_extract = "-xzf"
cmd_pack = "-czvf"
cmd = [tar_binary]
# numeric UIDs and GIDs are used, ACLs are enabled, SELinux is enabled
tar_options = ["--numeric-owner", "--acls", "--selinux"]
# used for packing directories into tarball
os.chdir('/root')
tarball_dir = get_tarball_name(result_file, current_time)
tarball_name = tarball_dir + '.tar.gz'
bkp_tar_dir = os.path.join('/root', tarball_dir)
if direction:
shutil.copytree(dirname, bkp_tar_dir, symlinks=True)
tarball = get_tarball_result_path(dirname, tarball_name)
cmd.append(cmd_pack)
cmd.append(tarball)
cmd.append(tarball_dir)
else:
cmd.append(cmd_extract)
cmd.append(result_file)
cmd.extend(tar_options)
run_subprocess(cmd, print_output=quiet)
shutil.rmtree(bkp_tar_dir)
if direction:
try:
shutil.copy(tarball,
os.path.join(settings.tarball_result_dir + "/"))
except IOError:
log_message(
"Problem with copying tarball {0} to /root/preupgrade-results".
format(tarball))
os.chdir(current_dir)
return os.path.join(settings.tarball_result_dir, tarball_name)
def generate(self):
if not self.collect_data():
log_message("Important data are missing for kickstart generation.",
level=logging.ERROR)
return None
packages = self.output_packages()
if packages:
self.ks.handler.packages.add(packages)
self.ks.handler.packages.handleMissing = KS_MISSING_IGNORE
self.update_repositories(self.repos)
self.update_users(self.filter_kickstart_users())
self.get_partition_layout('lsblk_list', 'vgs_list', 'lvdisplay')
self.update_partitioning()
self.embed_script(self.latest_tarball)
self.delete_obsolete_issues()
self.save_kickstart()
return True
def copy_modified_config_files(result_dir):
"""
Function copies all modified files to dirtyconf directory.
(files which are not mentioned in cleanconf directory)
"""
etc_va_log = os.path.join(settings.cache_dir, settings.common_name,
"rpm_etc_Va.log")
try:
lines = get_file_content(etc_va_log, "rb", method=True)
except IOError:
return
dirty_conf = os.path.join(result_dir, settings.dirty_conf_dir)
clean_conf = os.path.join(result_dir, settings.clean_conf_dir)
for line in lines:
try:
(opts, flags, filename) = line.strip().split()
except ValueError:
return
log_message("File name to copy '%s'" % filename,
print_output=0,
level=logging.INFO)
new_filename = filename[1:]
# Check whether config file exists in cleanconf directory
file_name = os.path.join(clean_conf, new_filename)
if os.path.exists(file_name):
message = "Configuration file '%s' exists in '%s' directory"
log_message(message % (file_name, clean_conf),
print_output=0,
level=logging.INFO)
continue
dirty_path = os.path.join(dirty_conf, os.path.dirname(new_filename))
# Check whether dirtyconf directory with dirname(filename) exists
if not os.path.exists(dirty_path):
os.makedirs(dirty_path)
# Copy filename to dirtyconf directory
try:
target_name = os.path.join(dirty_conf, new_filename)
if os.path.islink(filename):
filename = os.path.realpath(filename)
if os.path.exists(target_name):
log_message("File '%s' already exists in dirtyconf directory" %
target_name,
print_output=0,
level=logging.INFO)
continue
shutil.copyfile(filename, target_name)
except shutil.Error:
log_message("Copying file '%s' to '%s' failed." %
(filename, target_name),
print_output=0,
level=logging.INFO)
continue
except IOError:
continue
def run_third_party_modules(self, dir_name):
"""
Functions executes a 3rd party contents
3rd party contents are stored in
/usr/share/preupgrade/RHEL6_7/3rdparty directory
"""
for self.third_party, content in six.iteritems(list_contents(dir_name)):
third_party_name = self.third_party
log_message("Execution {0} assessments:".format(self.third_party))
self.report_parser.reload_xml(content)
self.content = content
self.run_scan_process()
self.report_data[third_party_name] = self.scanning_progress.get_output_data()
# This function prepare XML and generate HTML
self.prepare_xml_for_html()
self.third_party = ""
def get_volume_info(filename, first_index, second_index):
try:
volume_list = get_file_content(os.path.join(
settings.KS_DIR, filename),
'rb',
method=True,
decode_flag=False)
except IOError:
log_message(
"File %s is missing. Partitioning layout has not to be complete."
% filename,
level=logging.WARNING)
return None
volume_info = {}
for line in volume_list:
fields = line.strip().split(':')
volume_info[fields[first_index]] = fields[second_index]
return volume_info
def embed_script(self, tarball):
tarball_content = get_file_content(tarball, 'rb', decode_flag=False)
tarball_name = os.path.splitext(os.path.splitext(os.path.basename(tarball))[0])[0]
script_str = ''
try:
script_path = settings.KS_TEMPLATE_POSTSCRIPT
except AttributeError:
log_message('KS_TEMPLATE_POSTSCRIPT is not defined in settings.py')
return
script_str = get_file_content(os.path.join(settings.KS_DIR, script_path), 'rb')
if not script_str:
log_message("Can't open script template: {0}".format(script_path))
return
script_str = script_str.replace('{tar_ball}', base64.b64encode(tarball_content))
script_str = script_str.replace('{RESULT_NAME}', tarball_name)
script = Script(script_str, type=KS_SCRIPT_POST, inChroot=True)
self.ks.handler.scripts.append(script)
def tarball_result_dir(result_file, dirname, quiet, direction=True):
"""
pack results to tarball
direction is used as a flag for packing or extracting
For packing True
For unpacking False
"""
current_dir = os.getcwd()
tar_binary = "/bin/tar"
current_time = get_current_time()
cmd_extract = "-xzf"
cmd_pack = "-czvf"
cmd = [tar_binary]
# numeric UIDs and GIDs are used, ACLs are enabled, SELinux is enabled
tar_options = ["--numeric-owner", "--acls", "--selinux"]
# used for packing directories into tarball
os.chdir("/root")
tarball_dir = get_tarball_name(result_file, current_time)
tarball_name = tarball_dir + ".tar.gz"
bkp_tar_dir = os.path.join("/root", tarball_dir)
if direction:
shutil.copytree(dirname, bkp_tar_dir, symlinks=True)
tarball = get_tarball_result_path(dirname, tarball_name)
cmd.append(cmd_pack)
cmd.append(tarball)
cmd.append(tarball_dir)
else:
cmd.append(cmd_extract)
cmd.append(result_file)
cmd.extend(tar_options)
run_subprocess(cmd, print_output=quiet)
shutil.rmtree(bkp_tar_dir)
if direction:
try:
shutil.copy(tarball, os.path.join(settings.tarball_result_dir + "/"))
except IOError:
log_message("Problem with copying tarball {0} to /root/preupgrade-results".format(tarball))
os.chdir(current_dir)
return os.path.join(settings.tarball_result_dir, tarball_name)
def copy_modified_config_files(result_dir):
"""
Function copies all modified files to dirtyconf directory.
(files which are not mentioned in cleanconf directory)
"""
etc_va_log = os.path.join(settings.cache_dir, settings.common_name, "rpm_etc_Va.log")
try:
lines = get_file_content(etc_va_log, "rb", method=True)
except IOError:
return
dirty_conf = os.path.join(result_dir, settings.dirty_conf_dir)
clean_conf = os.path.join(result_dir, settings.clean_conf_dir)
for line in lines:
try:
(opts, flags, filename) = line.strip().split()
except ValueError:
return
log_message("File name to copy '%s'" % filename,
print_output=0,
level=logging.INFO)
new_filename = filename[1:]
# Check whether config file exists in cleanconf directory
file_name = os.path.join(clean_conf, new_filename)
if os.path.exists(file_name):
message = "Configuration file '%s' exists in '%s' directory"
log_message(message % (file_name, clean_conf),
print_output=0,
level=logging.INFO)
continue
dirty_path = os.path.join(dirty_conf, os.path.dirname(new_filename))
# Check whether dirtyconf directory with dirname(filename) exists
if not os.path.exists(dirty_path):
os.makedirs(dirty_path)
# Copy filename to dirtyconf directory
try:
target_name = os.path.join(dirty_conf, new_filename)
if os.path.islink(filename):
filename = os.path.realpath(filename)
if os.path.exists(target_name):
log_message("File '%s' already exists in dirtyconf directory" % target_name,
print_output=0,
level=logging.INFO)
continue
shutil.copyfile(filename, target_name)
except shutil.Error:
log_message("Copying file '%s' to '%s' failed." % (filename, target_name),
print_output=0,
level=logging.INFO)
continue
except IOError:
continue
def format_rules_to_table(output_data, content):
"""Function format output_data to table"""
if not output_data:
# If output_data does not contain anything then do not print nothing
return
max_title_length = max(x for x in [len(l.split(':')[0]) for l in output_data]) + 5
max_result_length = max(x for x in [len(l.split(':')[2]) for l in output_data]) + 2
log_message(settings.result_text.format(content))
message = '-' * (max_title_length + max_result_length + 4)
log_message(message)
for data in sorted(output_data, key=compare_data, reverse=True):
try:
title, dummy_rule_id, result = data.split(':')
except ValueError:
# data is not an information about processed test; let's log it as an error
log_message(data, level=logging.ERROR)
else:
log_message(u"|%s |%s|" % (title.ljust(max_title_length),
result.strip().ljust(max_result_length)))
log_message(message)
def get_interpreter(filename, verbose=False):
"""
The function returns interpreter
Checks extension of script and first line of script
"""
script_types = {
'/bin/bash': '.sh',
'/usr/bin/python': '.py',
'/usr/bin/perl': '.pl'
}
inter = list(k for k, v in six.iteritems(script_types)
if filename.endswith(v))
content = get_file_content(filename, 'rb')
if inter and content.startswith('#!' + inter[0]):
return inter
else:
if verbose:
log_message("Problem with getting interpreter",
level=logging.ERROR)
return None
def check_xml(xml_file):
"""
Check XML
return False if xml file is not okay or raise IOError if perms are
not okay; use python-magic to check the file if module is available
"""
if os.path.isfile(xml_file):
if not os.access(xml_file, os.R_OK):
log_message("File is not readable." % xml_file, level=logging.ERROR)
raise IOError("File %s is not readable." % xml_file)
else:
log_message("%s is not a file" % xml_file, level=logging.ERROR)
raise IOError("%s is not a file." % xml_file)
raw_test = False
is_valid = False
try:
import magic
except ImportError:
raw_test = True
else:
try:
xml_file_magic = magic.from_file(xml_file, mime=True)
except AttributeError:
raw_test = True
else:
is_valid = xml_file_magic == "application/xml"
if raw_test:
is_valid = xml_file.endswith(".xml")
if is_valid:
return xml_file
else:
log_message("Provided file is not a valid XML file", level=logging.ERROR)
raise RuntimeError("Provided file is not a valid XML file")
def show_progress(self, stdout_data):
"""Function shows a progress of assessment"""
try:
self.width_size = int(ScanProgress.get_terminal_width()[1])
except IndexError:
self.width_size = 80
xccdf_rule, dummy_result = stdout_data.strip().split(':')
self.output_data.append(u'{0}:{1}'.format(self.names[xccdf_rule],
stdout_data.strip()))
self.current_count += 1
old_width = self.width_size
self.width_size -= 21
prev_msg = self._return_correct_msg(self.get_full_name(self.current_count - 1))
self.width_size = old_width
cur_msg = self._return_correct_msg(self.get_full_name(self.current_count))
cnt_back = 7 + len(prev_msg) + 3
msg = u'%sdone (%s)' % ('\b' * cnt_back, prev_msg)
log_message(msg,
new_line=True,
log=False)
if self.total_count > self.current_count:
msg = self._return_correct_msg(u'%.3d/%.3d ...running (%s)' % (self.current_count + 1,
self.total_count,
cur_msg))
log_message(msg,
new_line=False,
log=False)
log_message(stdout_data.strip(), print_output=0)
def common_results(self):
"""run common scripts"""
log_message("Gathering logs used by preupgrade assistant:")
self.switch_dir()
try:
max_length = max(max([len(x.split("=", 4)[3]) for x in self.lines]), len(settings.assessment_text))
# Log files which will not be updated
# when RPM database is not changed
for counter, line in enumerate(self.lines):
line = line.strip()
if line.startswith("#"):
continue
cmd, log_file, dummy_bash_value, name, values = line.split("=", 4)
log_message("%s : %.2d/%d ...running" % (name.ljust(max_length),
counter+1,
len(self.lines)),
new_line=False, log=False)
start_time = datetime.datetime.now()
common_file_path = self.common_logfiles(log_file)
utils.run_subprocess(cmd, output=common_file_path, shell=True)
end_time = datetime.datetime.now()
diff = end_time - start_time
log_message(" %sfinished (time %.2d:%.2ds)" % ('\b' * 8,
diff.seconds / 60,
diff.seconds % 60),
log=False)
# os.chmod(common_file_path, 0640)
self.switch_back_dir()
except IOError:
return 0
else:
return 1
def show_progress(self, stdout_data):
"""Function shows a progress of assessment"""
try:
self.width_size = int(ScanProgress.get_terminal_width()[1])
except IndexError:
self.width_size = 80
xccdf_rule, dummy_result = stdout_data.strip().split(':')
self.output_data.append(u'{0}:{1}'.format(self.names[xccdf_rule],
stdout_data.strip()))
self.current_count += 1
old_width = self.width_size
self.width_size -= 21
prev_msg = self._return_correct_msg(
self.get_full_name(self.current_count - 1))
self.width_size = old_width
cur_msg = self._return_correct_msg(
self.get_full_name(self.current_count))
cnt_back = 7 + len(prev_msg) + 3
msg = u'%sdone (%s)' % ('\b' * cnt_back, prev_msg)
log_message(msg, new_line=True, log=False)
if self.total_count > self.current_count:
msg = self._return_correct_msg(
u'%.3d/%.3d ...running (%s)' %
(self.current_count + 1, self.total_count, cur_msg))
log_message(msg, new_line=False, log=False)
log_message(stdout_data.strip(), print_output=0)
def common_results(self):
"""run common scripts"""
log_message("Gathering logs used by preupgrade assistant:")
self.switch_dir()
try:
max_length = max(
max([len(x.split("=", 4)[3]) for x in self.lines]),
len(settings.assessment_text))
# Log files which will not be updated
# when RPM database is not changed
for counter, line in enumerate(self.lines):
line = line.strip()
if line.startswith("#"):
continue
cmd, log_file, dummy_bash_value, name, values = line.split(
"=", 4)
log_message(
"%s : %.2d/%d ...running" %
(name.ljust(max_length), counter + 1, len(self.lines)),
new_line=False,
log=False)
start_time = datetime.datetime.now()
common_file_path = self.common_logfiles(log_file)
utils.run_subprocess(cmd, output=common_file_path, shell=True)
end_time = datetime.datetime.now()
diff = end_time - start_time
log_message(" %sfinished (time %.2d:%.2ds)" %
('\b' * 8, diff.seconds / 60, diff.seconds % 60),
log=False)
# os.chmod(common_file_path, 0640)
self.switch_back_dir()
except IOError:
return 0
else:
return 1
def embed_script(self, tarball):
tarball_content = get_file_content(tarball, 'rb', decode_flag=False)
tarball_name = os.path.splitext(
os.path.splitext(os.path.basename(tarball))[0])[0]
script_str = ''
try:
script_path = settings.KS_TEMPLATE_POSTSCRIPT
except AttributeError:
log_message('KS_TEMPLATE_POSTSCRIPT is not defined in settings.py')
return
script_str = get_file_content(
os.path.join(settings.KS_DIR, script_path), 'rb')
if not script_str:
log_message("Can't open script template: {0}".format(script_path))
return
script_str = script_str.replace('{tar_ball}',
base64.b64encode(tarball_content))
script_str = script_str.replace('{RESULT_NAME}', tarball_name)
script = Script(script_str, type=KS_SCRIPT_POST, inChroot=True)
self.ks.handler.scripts.append(script)
def get_partition_layout(self, lsblk, vgs, lvdisplay):
"""
Returns dictionary with partition and realname and size
:param filename: filename with partition_layout in /root/preupgrade/kickstart directory
:return: dictionary with layout
"""
lsblk_filename = os.path.join(settings.KS_DIR, lsblk)
try:
layout = get_file_content(lsblk_filename, 'rb', method=True, decode_flag=False)
except IOError:
log_message("File %s was not generated by a content. Kickstart does not contain partitioning layout" % lsblk_filename)
self.part_layout = None
return None
vg_info = []
lv_info = []
if vgs is not None:
vg_info = KickstartGenerator.get_volume_info(vgs, 0, 5)
if lvdisplay is not None:
lv_info = KickstartGenerator.get_volume_info(lvdisplay, 0, 1)
pg = PartitionGenerator(layout, vg_info, lv_info)
pg.generate_partitioning()
self.part_layout.extend(pg.get_partitioning())
def format_rules_to_table(output_data, content):
"""Function format output_data to table"""
if not output_data:
# If output_data does not contain anything then do not print nothing
return
max_title_length = max(
x for x in [len(l.split(':')[0]) for l in output_data]) + 5
max_result_length = max(
x for x in [len(l.split(':')[2]) for l in output_data]) + 2
log_message(settings.result_text.format(content))
message = '-' * (max_title_length + max_result_length + 4)
log_message(message)
for data in sorted(output_data, key=compare_data, reverse=True):
try:
title, dummy_rule_id, result = data.split(':')
except ValueError:
# data is not an information about processed test; let's log it as an error
log_message(data, level=logging.ERROR)
else:
log_message(u"|%s |%s|" %
(title.ljust(max_title_length),
result.strip().ljust(max_result_length)))
log_message(message)
def load_or_default(system_ks_path):
"""load system ks or default ks"""
ksparser = KickstartParser(makeVersion())
try:
ksparser.readKickstart(system_ks_path)
except (KickstartError, IOError):
log_message("Can't read system kickstart at {0}".format(system_ks_path))
try:
ksparser.readKickstart(settings.KS_TEMPLATE)
except AttributeError:
log_message("There is no KS_TEMPLATE_POSTSCRIPT specified in settings.py")
except IOError:
log_message("Can't read kickstart template {0}".format(settings.KS_TEMPLATE))
return None
return ksparser
def load_or_default(system_ks_path):
"""load system ks or default ks"""
ksparser = KickstartParser(makeVersion())
try:
ksparser.readKickstart(system_ks_path)
except (KickstartError, IOError):
log_message(
"Can't read system kickstart at {0}".format(system_ks_path))
try:
ksparser.readKickstart(settings.KS_TEMPLATE)
except AttributeError:
log_message(
"There is no KS_TEMPLATE_POSTSCRIPT specified in settings.py"
)
except IOError:
log_message("Can't read kickstart template {0}".format(
settings.KS_TEMPLATE))
return None
return ksparser
def hash_postupgrade_file(verbose, dirname, check=False):
"""
The function creates hash file over all scripts in postupgrade.d directory.
In case of remediation it checks whether checksums are different and
print what scripts were changed.
"""
if not os.path.exists(dirname):
message = 'Directory %s does not exist for creating checksum file'
log_message(message, settings.postupgrade_dir, level=logging.ERROR)
return
postupg_scripts = get_all_postupgrade_files(verbose, dirname)
if not postupg_scripts:
return
filename = settings.base_hashed_file
if check:
filename = settings.base_hashed_file + "_new"
lines = []
for post_name in postupg_scripts:
lines.append(post_name + "=" + get_hash_file(post_name, sha1()) + "\n")
full_path_name = os.path.join(dirname, filename)
write_to_file(full_path_name, "wb", lines)
if check:
hashed_file = get_hashes(
os.path.join(dirname, settings.base_hashed_file))
if hashed_file is None:
message = 'Hashed_file is missing. Postupgrade scripts will not be executed'
log_message(message, level=logging.WARNING)
return False
hashed_file_new = get_hashes(full_path_name)
different_hashes = list(
set(hashed_file).difference(set(hashed_file_new)))
for file_name in [settings.base_hashed_file, filename]:
os.remove(os.path.join(dirname, file_name))
if different_hashes or len(different_hashes) > 0:
message = 'Checksums are different in these postupgrade scripts: %s'
log_message(message % different_hashes, level=logging.WARNING)
return False
return True
def hash_postupgrade_file(verbose, dirname, check=False):
"""
The function creates hash file over all scripts in postupgrade.d directory.
In case of remediation it checks whether checksums are different and
print what scripts were changed.
"""
if not os.path.exists(dirname):
message = 'Directory %s does not exist for creating checksum file'
log_message(message, settings.postupgrade_dir, level=logging.ERROR)
return
postupg_scripts = get_all_postupgrade_files(verbose, dirname)
if not postupg_scripts:
return
filename = settings.base_hashed_file
if check:
filename = settings.base_hashed_file + "_new"
lines = []
for post_name in postupg_scripts:
lines.append(post_name + "=" + get_hash_file(post_name, sha1())+"\n")
full_path_name = os.path.join(dirname, filename)
write_to_file(full_path_name, "wb", lines)
if check:
hashed_file = get_hashes(os.path.join(dirname, settings.base_hashed_file))
if hashed_file is None:
message = 'Hashed_file is missing. Postupgrade scripts will not be executed'
log_message(message, level=logging.WARNING)
return False
hashed_file_new = get_hashes(full_path_name)
different_hashes = list(set(hashed_file).difference(set(hashed_file_new)))
for file_name in [settings.base_hashed_file, filename]:
os.remove(os.path.join(dirname, file_name))
if different_hashes or len(different_hashes) > 0:
message = 'Checksums are different in these postupgrade scripts: %s'
log_message(message % different_hashes, level=logging.WARNING)
return False
return True
def run_scan_process(self):
"""Function scans the source system"""
self.xml_mgr = xml_manager.XmlManager(self.conf.result_dir,
self.get_scenario(),
os.path.basename(self.content),
self.conf.result_name)
self.report_parser.add_global_tags(self.conf.result_dir,
self.get_proper_scenario(self.get_scenario()),
self.conf.mode,
self._devel_mode,
self._dist_mode)
self.report_parser.modify_result_path(self.conf.result_dir,
self.get_proper_scenario(self.get_scenario()),
self.conf.mode)
# Execute assessment
self.scanning_progress = ScanProgress(self.get_total_check(), self.conf.debug)
self.scanning_progress.set_names(self.report_parser.get_name_of_checks())
log_message('%s:' % settings.assessment_text,
new_line=True,
log=False)
log_message('%.3d/%.3d ...running (%s)' % (
1,
self.get_total_check(),
self.scanning_progress.get_full_name(0)),
new_line=False,
log=False)
start_time = datetime.datetime.now()
self.run_scan(function=self.scanning_progress.show_progress)
end_time = datetime.datetime.now()
diff = end_time - start_time
log_message(
"Assessment finished (time %.2d:%.2ds)" % (diff.seconds/60,
diff.seconds%60),
log=False
)
def check_xml(xml_file):
"""
Check XML
return False if xml file is not okay or raise IOError if perms are
not okay; use python-magic to check the file if module is available
"""
if os.path.isfile(xml_file):
if not os.access(xml_file, os.R_OK):
log_message("File is not readable." % xml_file,
level=logging.ERROR)
raise IOError("File %s is not readable." % xml_file)
else:
log_message("%s is not a file" % xml_file, level=logging.ERROR)
raise IOError("%s is not a file." % xml_file)
raw_test = False
is_valid = False
try:
import magic
except ImportError:
raw_test = True
else:
try:
xml_file_magic = magic.from_file(xml_file, mime=True)
except AttributeError:
raw_test = True
else:
is_valid = xml_file_magic == 'application/xml'
if raw_test:
is_valid = xml_file.endswith(".xml")
if is_valid:
return xml_file
else:
log_message("Provided file is not a valid XML file",
level=logging.ERROR)
raise RuntimeError("Provided file is not a valid XML file")
def upload_results(self, tarball_path=None):
"""upload tarball with results to frontend"""
import xmlrpclib
import socket
if self.conf.upload is True:
# lets try default configuration
url = "http://127.0.0.1:8099/submit/"
else:
url = self.conf.upload \
if self.conf.upload[-1] == '/' \
else self.conf.upload + '/'
try:
proxy = xmlrpclib.ServerProxy(url)
proxy.submit.ping()
except Exception:
raise Exception('Can\'t connect to preupgrade assistant WEB-UI at %s.\n\n'
'Please ensure that package preupgrade-assistant-ui '
'has been installed on target system and firewall is set up '
'to allow connections on port 8099.' % url)
tarball_results = self.conf.results or tarball_path
file_content = get_file_content(tarball_results, 'rb', False, False)
binary = xmlrpclib.Binary(file_content)
host = socket.gethostname()
response = proxy.submit.submit_new({
'data': binary,
'host': host,
})
try:
status = response['status']
except KeyError:
log_message('Invalid response from server.')
log_message("Invalid response from server: %s" % response, level=logging.ERROR)
else:
if status == 'OK':
try:
url = response['url']
except KeyError:
log_message('Report submitted successfully.')
else:
log_message('Report submitted successfully. You can inspect it at %s' % url)
else:
try:
message = response['message']
log_message('Report not submitted. Server returned message: ', message)
log_message("Report submit: %s (%s)" % (status, message), level=logging.ERROR)
except KeyError:
log_message('Report not submitted. Server returned status: ', status)
log_message("Report submit: %s" % status, level=logging.ERROR)
def fault_repr(self):
"""monkey patching Fault's repr method so newlines are actually interpreted"""
log_message(self.faultString)
return "<Fault %s: %s>" % (self.faultCode, self.faultString)
def scan_system(self):
"""The function is used for scanning system with all steps."""
self._set_devel_mode()
self.prepare_scan_system()
assessment_dir = self.generate_report()
# Update source XML file in temporary directory
self.content = os.path.join(assessment_dir, settings.content_file)
try:
self.report_parser = ReportParser(self.content)
except IOError:
log_message("Content {0} does not exist".format(self.content))
sys.exit(1)
if not self.conf.contents:
version = get_assessment_version(self.conf.scan)
if version is None:
log_message("Your scan have wrong format",
level=logging.ERROR)
log_message("Examples format is like RHEL6_7",
level=logging.ERROR)
sys.exit(1)
self.report_parser.modify_platform_tag(version[0])
if self.conf.mode:
try:
lines = [i.rstrip() for i in get_file_content(os.path.join(os.path.dirname(self.path),
self.conf.mode),
'rb',
method=True)]
except IOError:
return
self.report_parser.select_rules(lines)
if self.conf.select_rules:
lines = [i.strip() for i in self.conf.select_rules.split(',')]
unknown_rules = self.report_parser.check_rules(lines)
if unknown_rules:
log_message(settings.unknown_rules % '\n'.join(unknown_rules))
self.report_parser.select_rules(lines)
self.run_scan_process()
main_report = self.scanning_progress.get_output_data()
# This function prepare XML and generate HTML
self.prepare_xml_for_html()
third_party_dir_name = self.get_third_party_dir(assessment_dir)
if os.path.exists(third_party_dir_name):
self.run_third_party_modules(third_party_dir_name)
self.copy_preupgrade_scripts(assessment_dir)
# It prints out result in table format
format_rules_to_table(main_report, "main contents")
for target, report in six.iteritems(self.report_data):
format_rules_to_table(report, "3rdparty content " + target)
tar_ball_name = tarball_result_dir(self.conf.tarball_name, self.conf.result_dir, self.conf.verbose)
log_message("Tarball with results is stored here %s ." % tar_ball_name)
log_message("The latest assessment is stored in directory %s ." % self.conf.result_dir)
# pack all configuration files to tarball
return tar_ball_name
def summary_report(self, tarball_path):
"""Function prints a summary report"""
command = settings.ui_command.format(tarball_path)
if self.conf.text:
path = self.get_default_txt_result_path()
else:
path = self.get_default_html_result_path()
report_dict = {
0: settings.message.format(path),
1: settings.message.format(path),
2: 'We found some critical issues. In-place upgrade is not advised.\n' +
"Read the file {0} for more details.".
format(path)
}
return_value = xccdf.check_inplace_risk(self.get_default_xml_result_path(), 0)
try:
if report_dict[int(return_value)]:
log_message('Summary information:')
log_message(report_dict[int(return_value)])
for report_type in settings.REPORTS:
file_name = settings.result_name + '-' + report_type + '.html'
report_name = os.path.join(os.path.dirname(self.report_parser.get_path()), file_name)
if os.path.exists(report_name):
log_message("Read the %s report file %s for more details." % (report_type, report_name))
except KeyError:
# We do not want to print anything in case of testing contents
pass
if self.report_data:
log_message('Summary 3rd party providers:')
for target, dummy_report in six.iteritems(self.report_data):
self.third_party = target
log_message("Read the 3rd party content {0} {1} for more details.".
format(target, path))
log_message("Upload results to UI by command:\ne.g. {0} .".format(command))
def run(self):
"""run analysis"""
if self.conf.version:
print ("Preupgrade Assistant version: %s" % VERSION)
return 0
if not self.conf.scan and not self.conf.contents:
cnt = 0
is_dir = lambda x: os.path.isdir(os.path.join(self.conf.source_dir, x))
dirs = os.listdir(self.conf.source_dir)
for dir_name in filter(is_dir, dirs):
if utils.get_assessment_version(dir_name):
self.conf.scan = dir_name
cnt += 1
if int(cnt) < 1:
log_message("There were no contents found in directory %s. \
If you would like to use this tool, you have to install some." % settings.source_dir)
return 1
if int(cnt) > 1:
log_message("Preupgrade assistant detects more then 1 set of contents in directory%s. \
If you would like to use this tool, you have to specify correct upgrade path parameter like -s RHEL6_7." % settings.source_dir)
return 1
if self.conf.list_contents_set:
for dir_name, dummy_content in six.iteritems(list_contents(self.conf.source_dir)):
log_message("{0}".format(dir_name))
return 0
if self.conf.list_rules:
log_message(settings.list_rules % '\n'.join(utils.get_list_rules(self.conf.scan)))
return 0
if self.conf.upload and self.conf.results:
self.upload_results()
return 0
if self.conf.mode and self.conf.select_rules:
log_message(settings.options_not_allowed)
return 1
if not self.conf.riskcheck and not self.conf.apply and not self.conf.cleanup and not self.conf.kickstart:
# If force option is not mentioned and user select NO then exits
if not self.conf.force and not show_message(settings.warning_text):
# We do not want to continue
return 0
if self.conf.text:
# Test whether w3m, lynx and elinks packages are installed
found = False
for pkg in utils.get_convertors():
if xml_manager.get_package_version(pkg):
self.text_convertor = pkg
found = True
break
if not found:
log_message(settings.converter_message.format(' '.join(utils.get_convertors())))
return 0
if os.geteuid() != 0:
print("Need to be root", end="\n")
if not self.conf.debug:
return 2
if self.conf.cleanup:
self.clean_preupgrade_environment()
sys.exit(0)
if self.conf.riskcheck:
return_val = xccdf.check_inplace_risk(self.get_default_xml_result_path(), self.conf.verbose)
return return_val
if self.conf.kickstart:
if not os.path.exists(self.get_default_xml_result_path()):
log_message("'preupg' command was not run yet. Run them before kickstart generation.")
return 1
kg = KickstartGenerator(settings.KS_DIR, self.get_preupgrade_kickstart())
KickstartGenerator.copy_kickstart_templates()
dummy_ks = kg.generate()
if dummy_ks:
log_message(settings.kickstart_text % self.get_preupgrade_kickstart())
return 0
if self.conf.scan:
self.content = os.path.join(self.conf.source_dir,
self.conf.scan,
settings.content_file)
if self.conf.scan.startswith("/"):
log_message('Specify correct upgrade path parameter like -s RHEL6_7')
log_message('Upgrade path is provided by command preupg --list')
return 1
if not os.path.isdir(os.path.join(self.conf.source_dir, self.conf.scan)):
log_message('Specify correct upgrade path parameter like -s RHEL6_7')
log_message('Upgrade path is provided by command preupg --list')
return 1
if self.conf.contents:
self.content = os.path.join(os.getcwd(), self.conf.contents)
# From content path like content-users/RHEL6_7 we need
# to get content-users dir
content_dir = self.conf.contents[:self.conf.contents.find(self.get_scenario())]
self.conf.source_dir = os.path.join(os.getcwd(), content_dir)
self.common = Common(self.conf)
if not self.conf.skip_common:
if not self.common.common_results():
return 1
if self.conf.scan or self.conf.contents:
if not os.path.exists(self.binary):
log_message("Oscap with SCE enabled is not installed")
return 1
if not os.access(self.binary, os.X_OK):
log_message("Oscap with SCE %s is not executable" % self.binary)
return 1
current_dir = os.getcwd()
os.chdir("/tmp")
tarball_path = self.scan_system()
self.summary_report(tarball_path)
self.common.copy_common_files()
KickstartGenerator.kickstart_scripts()
utils.remove_home_issues()
if self.conf.upload:
self.upload_results(tarball_path)
os.chdir(current_dir)
return 0
log_message('Nothing to do. Give me a task, please.')
self.conf.settings[2].parser.print_help()
return 0