def get_and_print_inplace_risk(verbose, inplace_risk):
"""
The function browse throw the list and find first
inplace_risk and return corresponding status.
If verbose mode is used then it prints out
all inplace risks higher then SLIGHT.
"""
risks = {
'SLIGHT:': 0,
'MEDIUM:': 0,
'HIGH:': 1,
'EXTREME:': 2,
}
return_value = -1
for key, val in sorted(iter(risks.items()), key=itemgetter(1),
reverse=False):
matched = [x for x in inplace_risk if key in x]
logger_report.debug(matched)
if matched:
# if matched and return_value the remember her
if return_value < val:
return_value = val
# If verbose mode is used and value is bigger then 0 then
# prints out
if int(verbose) > 1:
log_message('\n'.join(matched))
elif int(verbose) == 1 and val > 0:
log_message('\n'.join(matched))
return return_value
def comment_kickstart_issues(self):
"""Comment out those line in kickstart that are to be reviewed
by user and uncommented by themself if necessary.
"""
list_issues = [
' --', 'group', 'user ', 'repo', 'url', 'rootpw', 'part',
'volgroup', 'logvol', 'raid'
]
kickstart_data = []
try:
kickstart_data = FileHelper.get_file_content(os.path.join(
settings.KS_DIR, self.kickstart_name),
'rb',
method=True,
decode_flag=False)
except IOError:
log_message(
"The %s file is missing. The partitioning layout might not be complete."
% self.kickstart_name,
level=logging.WARNING)
return None
for index, row in enumerate(kickstart_data):
tag = [com for com in list_issues if row.startswith(com)]
if tag:
kickstart_data[index] = "#" + row
FileHelper.write_to_file(self.kickstart_name, 'wb', kickstart_data)
def show_progress(self, stdout_data):
"""Function shows a progress of assessment"""
try:
self.width_size = int(ScanProgress.get_terminal_width()[1])
except IndexError:
self.width_size = 80
logger_report.debug(stdout_data.strip())
xccdf_rule = ""
dummy_result = ""
try:
xccdf_rule, dummy_result = stdout_data.strip().split(':')
except ValueError:
print(stdout_data)
return
self.output_data.append(u'{0}:{1}'.format(self.names[xccdf_rule],
stdout_data.strip()))
self.current_count += 1
old_width = self.width_size
self.width_size -= 21
prev_msg = self._return_correct_msg(
self.get_full_name(self.current_count - 1))
self.width_size = old_width
cur_msg = self._return_correct_msg(
self.get_full_name(self.current_count))
cnt_back = 7 + len(prev_msg) + 3
msg = u'%sdone (%s)' % ('\b' * cnt_back, prev_msg)
log_message(msg, new_line=True)
if self.total_count > self.current_count:
msg = self._return_correct_msg(
u'%.3d/%.3d ...running (%s)' %
(self.current_count + 1, self.total_count, cur_msg))
log_message(msg, new_line=False)
def embed_script(self, tarball):
if tarball is None:
return
tarball_content = None
if os.path.exists(tarball):
tarball_content = FileHelper.get_file_content(tarball,
'rb',
decode_flag=False)
tarball_name = os.path.splitext(
os.path.splitext(os.path.basename(tarball))[0])[0]
script_str = ''
try:
script_path = settings.KS_TEMPLATE_POSTSCRIPT
except AttributeError:
log_message(
'KS_TEMPLATE_POSTSCRIPT is not defined in settings.py.')
return
script_str = FileHelper.get_file_content(
os.path.join(settings.KS_DIR, script_path), 'rb')
if not script_str:
log_message(
"Cannot open the script template: {0}.".format(script_path))
return
if tarball_content is not None:
script_str = script_str.replace('{tar_ball}',
base64.b64encode(tarball_content))
script_str = script_str.replace('{RESULT_NAME}', tarball_name)
script_str = script_str.replace('{TEMPORARY_PREUPG_DIR}',
'/root/preupgrade')
script = Script(script_str, type=KS_SCRIPT_POST, inChroot=True)
self.ks.handler.scripts.append(script)
def show_progress(self, stdout_data):
"""Function shows a progress of assessment"""
try:
self.width_size = int(ScanProgress.get_terminal_width()[1])
except IndexError:
self.width_size = 80
logger_report.debug(stdout_data.strip())
xccdf_rule = ""
dummy_result = ""
try:
xccdf_rule, dummy_result = stdout_data.strip().split(':')
except ValueError:
print (stdout_data)
return
self.output_data.append(u'{0}:{1}'.format(self.names[xccdf_rule], stdout_data.strip()))
self.current_count += 1
old_width = self.width_size
self.width_size -= 21
prev_msg = self._return_correct_msg(self.get_full_name(self.current_count - 1))
self.width_size = old_width
cur_msg = self._return_correct_msg(self.get_full_name(self.current_count))
cnt_back = 7 + len(prev_msg) + 3
msg = u'%sdone (%s)' % ('\b' * cnt_back, prev_msg)
log_message(msg, new_line=True)
if self.total_count > self.current_count:
msg = self._return_correct_msg(u'%.3d/%.3d ...running (%s)' % (self.current_count + 1,
self.total_count,
cur_msg))
log_message(msg, new_line=False)
def format_rules_to_table(output_data, content):
"""Function format output_data to table"""
if not output_data:
# If output_data does not contain anything then do not print nothing
return
max_title_length = max(
x for x in [len(l.split(':')[0]) for l in output_data]) + 5
max_result_length = max(
x for x in [len(l.split(':')[2]) for l in output_data]) + 2
log_message(settings.result_text.format(content))
message = '-' * (max_title_length + max_result_length + 4)
log_message(message)
for data in sorted(output_data,
key=ScanningHelper.compare_data,
reverse=True):
try:
title, dummy_rule_id, result = data.split(':')
except ValueError:
# data is not an information about processed test; let's log it as an error
log_message(data, level=logging.ERROR)
else:
log_message(u"|%s |%s|" %
(title.ljust(max_title_length),
result.strip().ljust(max_result_length)))
log_message(message)
def get_all_postupgrade_files(dummy_verbose, dir_name):
"""Function gets all postupgrade files from dir_name"""
postupg_scripts = []
for root, dummy_sub_dirs, files in os.walk(dir_name):
# find all files in this directory
postupg_scripts.extend([os.path.join(root, x) for x in files])
if not postupg_scripts:
log_message("No postupgrade scripts available")
return postupg_scripts
def tarball_result_dir(result_file, dirname, quiet, direction=True):
"""
pack results to tarball
direction is used as a flag for packing or extracting
For packing True
For unpacking False
"""
current_dir = os.getcwd()
tar_binary = "/bin/tar"
current_time = get_current_time()
cmd_extract = "-xzf"
cmd_pack = "-czvf"
cmd = [tar_binary]
# numeric UIDs and GIDs are used, ACLs are enabled, SELinux is enabled
tar_options = ["--numeric-owner", "--acls", "--selinux"]
# used for packing directories into tarball
os.chdir('/root')
tarball_dir = TarballHelper._get_tarball_name(result_file, current_time)
tarball_name = tarball_dir + '.tar.gz'
bkp_tar_dir = os.path.join('/root', tarball_dir)
if direction:
if not os.path.exists(bkp_tar_dir):
os.makedirs(bkp_tar_dir)
for preupg_dir in settings.preupgrade_dirs:
shutil.copytree(os.path.join(dirname, preupg_dir),
os.path.join(bkp_tar_dir, preupg_dir),
symlinks=True)
files_to_copy = [settings.PREUPG_README]
for root, subdirs, files in os.walk(dirname):
for f in files:
if f.startswith("result"):
files_to_copy.append(f)
for f in files_to_copy:
shutil.copyfile(os.path.join(dirname, f),
os.path.join(bkp_tar_dir, f))
tarball = TarballHelper._get_tarball_result_path(dirname, tarball_name)
cmd.append(cmd_pack)
cmd.append(tarball)
cmd.append(tarball_dir)
else:
cmd.append(cmd_extract)
cmd.append(result_file)
cmd.extend(tar_options)
ProcessHelper.run_subprocess(cmd, print_output=quiet)
shutil.rmtree(bkp_tar_dir)
if direction:
try:
shutil.copy(tarball, settings.tarball_result_dir + "/")
except IOError:
log_message("Problem with copying the tarball '%s' to /root/preupgrade-results", tarball)
os.chdir(current_dir)
return os.path.join(settings.tarball_result_dir, tarball_name)
def get_volume_info(filename, first_index, second_index):
try:
volume_list = FileHelper.get_file_content(filename, 'rb', method=True, decode_flag=False)
except IOError:
log_message("The %s file is missing. The partitioning layout might not be complete." % filename, level=logging.WARNING)
return None
volume_info = {}
for line in volume_list:
fields = line.strip().split(':')
volume_info[fields[first_index]] = fields[second_index]
return volume_info
def check_or_create_temp_dir(temp_dir, mode=None):
"""Check if provided temp dir is valid."""
if os.path.isdir(temp_dir):
if not os.access(temp_dir, os.W_OK):
log_message("The %s directory is not writable." % temp_dir, level=logging.ERROR)
raise IOError("The %s directory is not writable." % temp_dir)
else:
os.makedirs(temp_dir)
if mode:
os.chmod(temp_dir, mode)
return temp_dir
def determine_module_set_location(self):
"""Get module set path, directory name and path to the all_xccdf.xml"""
if not self.conf.contents: # was the --contents CLI option used?
if not self.conf.scan: # and what about the --scan option?
self.module_set_dirname = self.get_default_module_set_dirname()
if not self.module_set_dirname:
return ReturnValues.SCENARIO
else:
self.module_set_dirname = self.conf.scan
self.module_set_path = os.path.join(self.conf.source_dir,
self.module_set_dirname)
if not os.path.isdir(self.module_set_path):
log_message("Module set '%s' is not installed.\nFor a list"
" of installed module sets, use -l option." %
self.module_set_dirname,
level=logging.ERROR)
return ReturnValues.SCENARIO
self.all_xccdf_xml_path = os.path.join(
self.module_set_path, settings.all_xccdf_xml_filename)
else:
self.all_xccdf_xml_path = os.path.abspath(self.conf.contents)
self.module_set_path = os.path.dirname(self.all_xccdf_xml_path)
self.module_set_dirname = os.path.basename(self.module_set_path)
if not self.module_set_dirname:
log_message("Unable to determine a module set directory name.",
level=logging.ERROR)
log_message(
"The directory name needs to be in format"
" 'RHELx_y' for upgrades from RHEL x to RHEL y.",
level=logging.ERROR)
return ReturnValues.SCENARIO
if not os.path.isfile(self.all_xccdf_xml_path):
log_message("'%s' does not exist." % self.all_xccdf_xml_path,
level=logging.ERROR)
return ReturnValues.SCENARIO
def determine_module_set_location(self):
"""Get module set path, directory name and path to the all_xccdf.xml"""
if not self.conf.contents: # was the --contents CLI option used?
if not self.conf.scan: # and what about the --scan option?
self.module_set_dirname = self.get_default_module_set_dirname()
if not self.module_set_dirname:
return ReturnValues.SCENARIO
else:
self.module_set_dirname = self.conf.scan
self.module_set_path = os.path.join(self.conf.source_dir,
self.module_set_dirname)
if not os.path.isdir(self.module_set_path):
log_message("Module set '%s' is not installed.\nFor a list"
" of installed module sets, use -l option."
% self.module_set_dirname, level=logging.ERROR)
return ReturnValues.SCENARIO
self.all_xccdf_xml_path = os.path.join(
self.module_set_path, settings.all_xccdf_xml_filename)
else:
self.all_xccdf_xml_path = os.path.abspath(self.conf.contents)
self.module_set_path = os.path.dirname(self.all_xccdf_xml_path)
self.module_set_dirname = os.path.basename(self.module_set_path)
if not self.module_set_dirname:
log_message("Unable to determine a module set directory name.",
level=logging.ERROR)
log_message("The directory name needs to be in format"
" 'RHELx_y' for upgrades from RHEL x to RHEL y.",
level=logging.ERROR)
return ReturnValues.SCENARIO
if not os.path.isfile(self.all_xccdf_xml_path):
log_message("'%s' does not exist." % self.all_xccdf_xml_path,
level=logging.ERROR)
return ReturnValues.SCENARIO
def _check_available_contents(self):
cnt = 0
is_dir = lambda x: os.path.isdir(os.path.join(self.conf.source_dir, x))
dirs = os.listdir(self.conf.source_dir)
self.list_scans = []
for dir_name in filter(is_dir, dirs):
if SystemIdentification.get_assessment_version(dir_name):
self.conf.scan = dir_name
self.list_scans.append(dir_name)
logger_debug.debug("Scan directory '%s'", self.conf.scan)
cnt += 1
if int(cnt) < 1:
log_message("There were no modules found in the %s directory. \
If you would like to use this tool, you have to install some." %
settings.source_dir)
return ReturnValues.SCENARIO
if int(cnt) > 1:
log_message("Preupgrade Assistant detects more "
"than one set of modules in the %s directory.\n" %
settings.source_dir)
log_message("The list of sets of all available modules is: \n%s" %
'\n'.join(self.list_scans))
log_message(
"If you would like to use the tool, "
"specify the correct upgrade path mentioned above with a parameter -s."
)
return ReturnValues.SCENARIO
return 0
def summary_report(self, tarball_path):
"""Function prints a summary report"""
command = settings.ui_command.format(tarball_path)
if self.conf.text:
path = self.openscap_helper.get_default_txt_result_path()
else:
path = self.openscap_helper.get_default_html_result_path()
report_dict = {
0: settings.risks_found_warning.format(path),
1: settings.risks_found_warning.format(path),
2: 'We have found some critical issues. In-place upgrade or migration is not advised.\n' +
"Read the file {0} for more details.".format(path),
3: 'We have found some error issues. In-place upgrade or migration is not advised.\n' +
"Read the file {0} for more details.".format(path)
}
report_return_value = XccdfHelper.check_inplace_risk(
self.openscap_helper.get_default_xml_result_path(),
0)
try:
if report_dict[int(report_return_value)]:
log_message('Summary information:')
log_message(report_dict[int(report_return_value)])
except KeyError:
# We do not want to print anything in case of testing contents
pass
if not self.conf.mode or self.conf.mode == "upgrade":
# User either has not specied mode (upgrade and migration both
# together by default) or has chosen upgrade only = print warning
# to backup the system before doing the in-place upgrade
log_message(settings.upgrade_backup_warning)
log_message("Upload results to UI by the command:\ne.g. {0} .".format(command))
return report_return_value
def prepare_scan_system(self):
"""Function cleans previous scan and creates relevant directories"""
# First of all we need to delete the older one assessment
self.clean_scan()
self.prepare_scan_directories()
scenario = self.get_scenario()
if scenario is None:
log_message('Invalid scenario: %s' % self.conf.contents)
return ReturnValues.SCENARIO
scenario_path = os.path.join(self.conf.source_dir, scenario)
if not os.path.isdir(scenario_path):
log_message('Invalid scenario: %s' % scenario, level=logging.ERROR)
return ReturnValues.SCENARIO
return 0
def get_volume_info(filename, first_index, second_index):
try:
volume_list = FileHelper.get_file_content(filename, 'rb',
method=True,
decode_flag=False)
except IOError:
log_message("The %s file is missing. The partitioning layout might"
" not be complete." % filename, level=logging.WARNING)
return None
volume_info = {}
for line in volume_list:
fields = line.strip().split(':')
volume_info[fields[first_index]] = fields[second_index]
return volume_info
def summary_report(self, tarball_path):
"""Function prints a summary report"""
command = settings.ui_command.format(tarball_path)
if self.conf.text:
path = self.openscap_helper.get_default_txt_result_path()
else:
path = self.openscap_helper.get_default_html_result_path()
report_dict = {
0: settings.risks_found_warning.format(path),
1: settings.risks_found_warning.format(path),
2: 'We have found some critical issues. In-place upgrade or migration is not advised.\n' +
"Read the file {0} for more details.".format(path),
3: 'We have found some error issues. In-place upgrade or migration is not advised.\n' +
"Read the file {0} for more details.".format(path)
}
report_return_value = XccdfHelper.check_inplace_risk(
self.openscap_helper.get_default_xml_result_path(),
0)
try:
if report_dict[int(report_return_value)]:
log_message('Summary information:')
log_message(report_dict[int(report_return_value)])
except KeyError:
# We do not want to print anything in case of testing contents
pass
if not self.conf.mode or self.conf.mode == "upgrade":
# User either has not specied mode (upgrade and migration both
# together by default) or has chosen upgrade only = print warning
# to backup the system before doing the in-place upgrade
log_message(settings.upgrade_backup_warning)
log_message("Upload results to UI by the command:\ne.g. {0} .".format(command))
return report_return_value
def postupgrade_scripts(verbose, dirname):
"""
The function runs postupgrade directory
If dir does not exists the report and return
"""
if not os.path.exists(dirname):
log_message('There is no any %s directory' % settings.postupgrade_dir,
level=logging.WARNING)
return
postupg_scripts = PostupgradeHelper.get_all_postupgrade_files(verbose, dirname)
if not postupg_scripts:
return
#max_length = max(list([len(x) for x in postupg_scripts]))
log_message('Running postupgrade scripts:')
for scr in sorted(postupg_scripts):
interpreter = FileHelper.get_interpreter(scr, verbose=verbose)
if interpreter is None:
continue
log_message('Executing script %s' % scr)
cmd = "{0} {1}".format(interpreter, scr)
ProcessHelper.run_subprocess(cmd, print_output=False, shell=True)
log_message("Executing script %s ...done" % scr)
def check_postmigrate_dir(self):
if not FileHelper.get_list_executable_files_in_dir(os.path.join(settings.assessment_results_dir,
settings.postmigrate_dir)):
if not self.conf.assumeyes:
accept = ['y', 'yes']
log_message("The '%s' folder is empty - scripts to be executed "
"after the migration should be placed "
"here." % os.path.join(settings.result_dir,
settings.postmigrate_dir))
message = "Do you want to continue with kickstart " \
"generation without any postmigration scripts?"
choice = MessageHelper.get_message(message=message, prompt="(Y/n)")
if choice.lower() not in accept:
return None
return True
def prepare_scan_system(self):
"""Function cleans previous scan and creates relevant directories"""
# First of all we need to delete the older one assessment
self.clean_scan()
self.prepare_scan_directories()
scenario = self.get_scenario()
if scenario is None:
log_message('Invalid scenario: %s' % self.conf.contents)
return ReturnValues.SCENARIO
scenario_path = os.path.join(self.conf.source_dir, scenario)
if not os.path.isdir(scenario_path):
log_message('Invalid scenario: %s' % scenario,
level=logging.ERROR)
return ReturnValues.SCENARIO
return 0
def copy_modified_config_files(result_dir):
"""
Function copies all modified files to dirtyconf directory.
(files which are not mentioned in cleanconf directory)
"""
etc_va_log = os.path.join(settings.cache_dir, settings.common_name,
"rpm_etc_Va.log")
try:
lines = FileHelper.get_file_content(etc_va_log, "rb", method=True)
except IOError:
raise IOError("Error: File that lists modified configuration files"
"'%s' is missing.\n" % etc_va_log)
dirty_conf = os.path.join(result_dir, settings.dirty_conf_dir)
clean_conf = os.path.join(result_dir, settings.clean_conf_dir)
# Go through all changed config files
for line in lines:
try:
(opts, flags, filename) = line.strip().split()
if opts.strip() == 'missing':
continue
except ValueError:
return
logger_debug.debug("The '%s' file name to copy.", filename)
new_filename = filename[1:]
# Check whether config file exists in cleanconf directory
cleanconf_file_name = os.path.join(clean_conf, new_filename)
dirtyconf_file_name = os.path.join(dirty_conf, new_filename)
# Check if config file does not exist in cleanconf directory
if ConfigFilesHelper.check_cleanconf_dir(result_dir,
cleanconf_file_name):
if os.path.exists(dirtyconf_file_name):
log_message(
"The %s file exist in the %s directory" %
(new_filename, dirty_conf), logging.DEBUG)
os.unlink(dirtyconf_file_name)
continue
# Check if config file does not exists in dirtyconf directory
check = ConfigFilesHelper.check_dirtyconf_dir(
dirtyconf_file_name, filename)
if check:
try:
shutil.copyfile(check, dirtyconf_file_name)
except IOError:
sys.stderr.write(
"Warning: Could not copy '%s' to '%s'.\n" %
(check, os.path.dirname(dirtyconf_file_name)))
pass
def get_interpreter(filename, verbose=False):
"""
The function returns interpreter
Checks extension of script and first line of script
"""
script_types = {'/bin/bash': '.sh',
'/usr/bin/python': '.py',
'/usr/bin/perl': '.pl'}
inter = list(k for k, v in six.iteritems(script_types) if filename.endswith(v))
content = FileHelper.get_file_content(filename, 'rb')
if inter and content.startswith('#!'+inter[0]):
return inter
else:
if verbose:
log_message("Problem with getting an interpreter", level=logging.ERROR)
return None
def main(self):
if not os.path.exists(os.path.join(settings.assessment_results_dir,
settings.xml_result_name)):
log_message("The 'preupg' command was not run yet. Run it before the Kickstart generation.")
return 1
KickstartGenerator.copy_kickstart_templates()
dummy_ks = self.generate()
if dummy_ks:
tar_ball_dir = os.path.basename(self.latest_tarball).split('.')[0]
kickstart_dir = os.path.join(os.path.dirname(self.dir_name),
tar_ball_dir)
log_message(settings.kickstart_text % (self.kickstart_name,
kickstart_dir,
kickstart_dir))
KickstartGenerator.kickstart_scripts()
def generate(self):
if not self.collect_data():
log_message("Important data are missing for the Kickstart generation.", level=logging.ERROR)
return None
if not self.check_postmigrate_dir():
return None
self.ks.handler.packages.excludedList = []
self.plugin_classes = self.load_plugins(os.path.dirname(__file__))
for module in six.iterkeys(self.plugin_classes):
self.plugin_classes[module].run_module()
self.ks.handler.packages.handleMissing = KS_MISSING_IGNORE
self.ks.handler.keyboard.keyboard = 'us'
self.embed_script(self.latest_tarball)
self.delete_obsolete_issues()
self.save_kickstart()
self.comment_kickstart_issues()
return True
def comment_kickstart_issues(self):
list_issues = [' --', 'group', 'user ', 'repo', 'url', 'rootpw']
kickstart_data = []
try:
kickstart_data = FileHelper.get_file_content(os.path.join(settings.KS_DIR, self.kickstart_name),
'rb',
method=True,
decode_flag=False)
except IOError:
log_message("The %s file is missing. The partitioning layout might not be complete." % self.kickstart_name,
level=logging.WARNING)
return None
for index, row in enumerate(kickstart_data):
tag = [com for com in list_issues if row.startswith(com)]
if tag:
kickstart_data[index] = "#" + row
FileHelper.write_to_file(self.kickstart_name, 'wb', kickstart_data)
def run_third_party_modules(self, dir_name):
"""
Functions executes a 3rd party contents
3rd party contents are stored in
/usr/share/preupgrade/RHEL6_7/3rdparty directory
"""
for third_party, content in six.iteritems(list_contents(dir_name)):
third_party_name = self.third_party = third_party
log_message("Execution {0} assessments:".format(third_party))
self.report_parser.reload_xml(content)
self.content = content
self.run_scan_process()
self.report_data[third_party_name] = self.scanning_progress.get_output_data()
# This function prepare XML and generate HTML
self.prepare_xml_for_html()
self.set_third_party("")
def _set_old_report_style(self):
"""
Choose which HTML report style should be used.
Set self.old_report_style to True when it is required from commandline
or older version of OpenSCAP has been detected.
"""
if self.conf.old_report_style:
self.old_report_style = True
elif OpenSCAPHelper.is_oscap_equal_or_greater(1, 2, 7) is False:
# in case that OpenSCAP version is lower then 1.2.7, fallback
# to the old (simple) report style.
log_message("Generating simply styled report due to the "
"limitations of the installed OpenSCAP")
self.old_report_style = True
else:
self.old_report_style = False
def _set_old_report_style(self):
"""
Choose which HTML report style should be used.
Set self.old_report_style to True when it is required from commandline
or older version of OpenSCAP has been detected.
"""
if self.conf.old_report_style:
self.old_report_style = True
elif OpenSCAPHelper.is_oscap_equal_or_greater(1, 2, 7) is False:
# in case that OpenSCAP version is lower then 1.2.7, fallback
# to the old (simple) report style.
log_message("Generating simply styled report due to the "
"limitations of the installed OpenSCAP")
self.old_report_style = True
else:
self.old_report_style = False
def scan_system(self):
"""The function is used for scanning system with all steps."""
self._set_devel_mode()
if not self.is_module_set_valid():
return ReturnValues.SCENARIO
ret_val = self.prepare_scan_system()
if ret_val != 0:
return ret_val
# Update source XML file in temporary directory
self.openscap_helper.update_variables(self.conf.assessment_results_dir,
self.conf.result_prefix,
self.conf.xml_result_name,
self.conf.html_result_name,
self.all_xccdf_xml_copy_path)
try:
self.report_parser = ReportParser(self.all_xccdf_xml_copy_path)
except IOError:
log_message("Error: Unable to open {0}.".format(
self.all_xccdf_xml_copy_path))
return ReturnValues.SCENARIO
if self.conf.mode:
lines = [
i.rstrip() for i in FileHelper.get_file_content(os.path.join(
self.module_set_copy_path, self.conf.mode),
'rb',
method=True)
]
self.report_parser.select_rules(lines)
if self.conf.select_rules:
lines = [i.strip() for i in self.conf.select_rules.split(',')]
unknown_rules = self.report_parser.check_rules(lines)
if unknown_rules:
log_message(settings.unknown_rules % '\n'.join(unknown_rules))
self.report_parser.select_rules(lines)
self.run_scan_process()
main_report = self.scanning_progress.get_output_data()
self.prepare_xml_for_html()
self.generate_html_or_text()
self.update_xml_after_html_generated()
self.copy_postupgrade_files()
self.copy_preupgrade_scripts(self.module_set_copy_path)
ConfigFilesHelper.copy_modified_config_files(
settings.assessment_results_dir)
# It prints out result in table format
ScanningHelper.format_rules_to_table(main_report, "main contents")
self.tar_ball_name = TarballHelper.tarball_result_dir(
self.conf.tarball_name, self.conf.verbose)
log_message("The tarball with results is stored in '%s' ." %
self.tar_ball_name)
log_message("The latest assessment is stored in the '%s' directory." %
self.conf.assessment_results_dir)
# pack all configuration files to tarball
return 0
def check_postmigrate_dir(self):
if not FileHelper.get_list_executable_files_in_dir(
os.path.join(settings.assessment_results_dir,
settings.postmigrate_dir)):
if not self.conf.assumeyes:
accept = ['y', 'yes']
log_message(
"The '%s' folder is empty - scripts to be executed "
"after the migration should be placed "
"here." % os.path.join(settings.result_dir,
settings.postmigrate_dir))
message = "Do you want to continue with kickstart " \
"generation without any postmigration scripts?"
choice = MessageHelper.get_message(message=message,
prompt="(Y/n)")
if choice.lower() not in accept:
return None
return True
def run_third_party_modules(self, dir_name):
"""
Functions executes a 3rd party contents
3rd party contents are stored in
/usr/share/preupgrade/RHEL6_7/3rdparty directory
"""
for third_party, content in six.iteritems(list_contents(dir_name)):
third_party_name = self.third_party = third_party
log_message("Execution {0} assessments:".format(third_party))
self.report_parser.reload_xml(content)
self.content = content
self.run_scan_process()
self.report_data[
third_party_name] = self.scanning_progress.get_output_data()
# This function prepare XML and generate HTML
self.prepare_xml_for_html()
self.set_third_party("")
def get_partition_layout(self, lsblk, vgs, lvdisplay):
"""
Returns dictionary with partition and realname and size
:param filename: filename with partition_layout in /root/preupgrade/kickstart directory
:return: dictionary with layout
"""
lsblk_filename = os.path.join(settings.KS_DIR, lsblk)
try:
self.layout = FileHelper.get_file_content(lsblk_filename, 'rb', method=True, decode_flag=False)
except IOError:
log_message("The %s file was not generated by the module. "
"Kickstart does not contain the partitioning layout" % lsblk_filename)
self.part_layout = None
return None
if vgs is not None:
self.vg_info = PartitionGenerator.get_volume_info(os.path.join(settings.KS_DIR, vgs), 0, 5)
if lvdisplay is not None:
self.lvdisplay = PartitionGenerator.get_volume_info(os.path.join(settings.KS_DIR, lvdisplay), 0, 1)
def replace_inplace_risk(self, scanning_results=None):
"""
This function has aim to replace FAILED to
NEEDS_INSPECTION in case that risks are SLIGHT or MEDIUM
"""
#Filter all rule-result in TestResult
changed_fields = []
self.remove_empty_check_import()
inplace_dict = {
0: ReportHelper.upd_inspection,
1: ReportHelper.upd_action,
2: ReportHelper.upd_extreme,
}
for rule in self.get_all_result_rules():
result = [
x for x in self.get_nodes(rule, "result") if x.text == "fail"
]
# Get all affected rules and taken their names
for res in result:
inplace_risk = XccdfHelper.get_check_import_inplace_risk(rule)
logger_report.debug(inplace_risk)
# In case that report has state fail and
# no log_risk than it should be needs_inspection
if not inplace_risk:
changed = rule.get("idref")
res.text = "fail"
log_message(
'The %s module exits as fail but without a risk.' %
rule.get("idref"))
else:
inplace_num = XccdfHelper.get_and_print_inplace_risk(
0, inplace_risk)
logger_report.debug("Call function '%s'",
inplace_dict[inplace_num])
changed, res.text = inplace_dict[inplace_num](rule)
logger_report.debug("Replace text '%s:%s'", changed,
res.text)
if changed is not None:
changed_fields.append(changed + ":" + res.text)
if scanning_results:
scanning_results.update_data(changed_fields)
self.write_xml()
def generate(self):
if not self.collect_data():
log_message(
"Important data are missing for the Kickstart generation.",
level=logging.ERROR)
return None
if not self.check_postmigrate_dir():
return None
self.ks.handler.packages.excludedList = []
self.plugin_classes = self.load_plugins(os.path.dirname(__file__))
for module in six.iterkeys(self.plugin_classes):
self.plugin_classes[module].run_module()
self.ks.handler.packages.handleMissing = KS_MISSING_IGNORE
self.ks.handler.keyboard.keyboard = 'us'
self.embed_script(self.latest_tarball)
self.delete_obsolete_issues()
self.save_kickstart()
self.comment_kickstart_issues()
return True
def main(self):
if not os.path.exists(
os.path.join(settings.assessment_results_dir,
settings.xml_result_name)):
log_message(
"The 'preupg' command was not run yet. Run it before the Kickstart generation."
)
return 1
KickstartGenerator.copy_kickstart_templates()
dummy_ks = self.generate()
if dummy_ks:
tar_ball_dir = os.path.basename(self.latest_tarball).split('.')[0]
kickstart_dir = os.path.join(os.path.dirname(self.dir_name),
tar_ball_dir)
log_message(settings.kickstart_text %
(self.kickstart_name, kickstart_dir, kickstart_dir))
KickstartGenerator.kickstart_scripts()
def copy_modified_config_files(result_dir):
"""
Function copies all modified files to dirtyconf directory.
(files which are not mentioned in cleanconf directory)
"""
etc_va_log = os.path.join(settings.cache_dir, settings.common_name, "rpm_etc_Va.log")
try:
lines = FileHelper.get_file_content(etc_va_log, "rb", method=True)
except IOError:
raise IOError("Error: File that lists modified configuration files"
"'%s' is missing.\n" % etc_va_log)
dirty_conf = os.path.join(result_dir, settings.dirty_conf_dir)
clean_conf = os.path.join(result_dir, settings.clean_conf_dir)
# Go through all changed config files
for line in lines:
try:
(opts, flags, filename) = line.strip().split()
if opts.strip() == 'missing':
continue
except ValueError:
return
logger_debug.debug("The '%s' file name to copy.", filename)
new_filename = filename[1:]
# Check whether config file exists in cleanconf directory
cleanconf_file_name = os.path.join(clean_conf, new_filename)
dirtyconf_file_name = os.path.join(dirty_conf, new_filename)
# Check if config file does not exist in cleanconf directory
if ConfigFilesHelper.check_cleanconf_dir(result_dir, cleanconf_file_name):
if os.path.exists(dirtyconf_file_name):
log_message("The %s file exist in the %s directory" % (new_filename, dirty_conf), logging.DEBUG)
os.unlink(dirtyconf_file_name)
continue
# Check if config file does not exists in dirtyconf directory
check = ConfigFilesHelper.check_dirtyconf_dir(dirtyconf_file_name, filename)
if check:
try:
shutil.copyfile(check, dirtyconf_file_name)
except IOError:
sys.stderr.write("Warning: Could not copy '%s' to '%s'.\n"
% (check,
os.path.dirname(dirtyconf_file_name)))
pass
def format_rules_to_table(output_data, content):
"""Function format output_data to table"""
if not output_data:
# If output_data does not contain anything then do not print nothing
return
max_title_length = max(x for x in [len(l.split(':')[0]) for l in output_data]) + 5
max_result_length = max(x for x in [len(l.split(':')[2]) for l in output_data]) + 2
log_message(settings.result_text.format(content))
message = '-' * (max_title_length + max_result_length + 4)
log_message(message)
for data in sorted(output_data, key=ScanningHelper.compare_data, reverse=True):
try:
title, dummy_rule_id, result = data.split(':')
except ValueError:
# data is not an information about processed test; let's log it as an error
log_message(data, level=logging.ERROR)
else:
log_message(u"|%s |%s|" % (title.ljust(max_title_length),
result.strip().ljust(max_result_length)))
log_message(message)
def get_partition_layout(self, lsblk, vgs, lvdisplay):
lsblk_filename = os.path.join(settings.KS_DIR, lsblk)
try:
self.layout = FileHelper.get_file_content(lsblk_filename, 'rb',
method=True,
decode_flag=False)
except IOError:
log_message("The %s file was not generated by the module. "
"Kickstart does not contain the partitioning layout"
% lsblk_filename)
self.part_layout = None
return None
if vgs is not None:
self.vg_info = PartitionGenerator.get_volume_info(
os.path.join(settings.KS_DIR, vgs), 0, 5
)
if lvdisplay is not None:
self.lvdisplay = PartitionGenerator.get_volume_info(
os.path.join(settings.KS_DIR, lvdisplay), 0, 1
)
def scan_system(self):
"""The function is used for scanning system with all steps."""
self._set_devel_mode()
if not self.is_module_set_valid():
return ReturnValues.SCENARIO
ret_val = self.prepare_scan_system()
if ret_val != 0:
return ret_val
# Update source XML file in temporary directory
self.openscap_helper.update_variables(self.conf.assessment_results_dir,
self.conf.result_prefix,
self.conf.xml_result_name,
self.conf.html_result_name,
self.all_xccdf_xml_copy_path)
try:
self.report_parser = ReportParser(self.all_xccdf_xml_copy_path)
except IOError:
log_message("Error: Unable to open {0}."
.format(self.all_xccdf_xml_copy_path))
return ReturnValues.SCENARIO
if self.conf.mode:
lines = [i.rstrip() for i in
FileHelper.get_file_content(
os.path.join(self.module_set_copy_path,
self.conf.mode),
'rb', method=True)]
self.report_parser.select_rules(lines)
if self.conf.select_rules:
lines = [i.strip() for i in self.conf.select_rules.split(',')]
unknown_rules = self.report_parser.check_rules(lines)
if unknown_rules:
log_message(settings.unknown_rules % '\n'.join(unknown_rules))
self.report_parser.select_rules(lines)
self.run_scan_process()
main_report = self.scanning_progress.get_output_data()
self.prepare_xml_for_html()
self.generate_html_or_text()
self.update_xml_after_html_generated()
self.copy_postupgrade_files()
self.copy_preupgrade_scripts(self.module_set_copy_path)
ConfigFilesHelper.copy_modified_config_files(
settings.assessment_results_dir)
# It prints out result in table format
ScanningHelper.format_rules_to_table(main_report, "main contents")
self.tar_ball_name = TarballHelper.tarball_result_dir(self.conf.tarball_name, self.conf.verbose)
log_message("The tarball with results is stored in '%s' ." % self.tar_ball_name)
log_message("The latest assessment is stored in the '%s' directory." % self.conf.assessment_results_dir)
# pack all configuration files to tarball
return 0
def common_results(self):
"""run common scripts"""
log_message("Gathering logs used by the Preupgrade Assistant:")
self.switch_dir()
try:
max_length = max(
max([len(x.split("=", 4)[3]) for x in self.lines]),
len(settings.assessment_text))
# Log files which will not be updated
# when RPM database is not changed
for counter, line in enumerate(self.lines):
line = line.strip()
if line.startswith("#"):
continue
cmd, log_file, dummy_bash_value, name, values = line.split(
"=", 4)
log_message(
"%s : %.2d/%d ...running" %
(name.ljust(max_length), counter + 1, len(self.lines)),
new_line=False)
start_time = datetime.datetime.now()
common_file_path = self.common_logfiles(log_file)
ProcessHelper.run_subprocess(cmd,
output=common_file_path,
shell=True)
end_time = datetime.datetime.now()
diff = end_time - start_time
log_message(" %sfinished (time %.2d:%.2ds)" %
('\b' * 8, diff.seconds / 60, diff.seconds % 60))
# os.chmod(common_file_path, 0640)
self.switch_back_dir()
except IOError:
return 0
else:
return 1
def run_scan_process(self):
"""Function scans the source system"""
self.xml_mgr = xml_manager.XmlManager(self.conf.assessment_results_dir,
self.get_scenario(),
os.path.basename(self.content),
self.conf.result_prefix)
self.report_parser.add_global_tags(self.conf.assessment_results_dir,
self.get_proper_scenario(self.get_scenario()),
self.conf.mode,
self._devel_mode,
self._dist_mode)
self.report_parser.modify_result_path(self.conf.assessment_results_dir,
self.get_proper_scenario(self.get_scenario()),
self.conf.mode)
# Execute assessment
self.scanning_progress = ScanProgress(self.get_total_check(), self.conf.debug)
self.scanning_progress.set_names(self.report_parser.get_name_of_checks())
log_message('%s:' % settings.assessment_text, new_line=True)
log_message('%.3d/%.3d ...running (%s)' % (
1,
self.get_total_check(),
self.scanning_progress.get_full_name(0)),
new_line=False
)
start_time = datetime.datetime.now()
self.run_scan(function=self.scanning_progress.show_progress)
end_time = datetime.datetime.now()
diff = end_time - start_time
log_message(
"The assessment finished (time %.2d:%.2ds)" % (diff.seconds / 60,
diff.seconds % 60)
)
def common_results(self):
"""run common scripts"""
log_message("Gathering logs used by the Preupgrade Assistant:")
self.switch_dir()
try:
max_length = max(max([len(x.split("=", 4)[3]) for x in self.lines]), len(settings.assessment_text))
# Log files which will not be updated
# when RPM database is not changed
for counter, line in enumerate(self.lines):
line = line.strip()
if line.startswith("#"):
continue
cmd, log_file, dummy_bash_value, name, values = line.split("=", 4)
log_message("%s : %.2d/%d ...running" % (name.ljust(max_length),
counter+1,
len(self.lines)),
new_line=False)
start_time = datetime.datetime.now()
common_file_path = self.common_logfiles(log_file)
ProcessHelper.run_subprocess(cmd, output=common_file_path, shell=True)
end_time = datetime.datetime.now()
diff = end_time - start_time
log_message(" %sfinished (time %.2d:%.2ds)" % ('\b' * 8,
diff.seconds / 60,
diff.seconds % 60))
# os.chmod(common_file_path, 0640)
self.switch_back_dir()
except IOError:
return 0
else:
return 1
def check_xml(xml_file):
"""
Check XML
return False if xml file is not okay or raise IOError if perms are
not okay; use python-magic to check the file if module is available
"""
if os.path.isfile(xml_file):
if not os.access(xml_file, os.R_OK):
log_message("The file is not readable." % xml_file, level=logging.ERROR)
raise IOError("The %s file is not readable." % xml_file)
else:
log_message("%s is not a file" % xml_file, level=logging.ERROR)
raise IOError("%s is not a file." % xml_file)
raw_test = False
is_valid = False
try:
import magic
except ImportError:
raw_test = True
else:
try:
xml_file_magic = magic.from_file(xml_file, mime=True)
except AttributeError:
raw_test = True
else:
is_valid = xml_file_magic == 'application/xml'
if raw_test:
is_valid = xml_file.endswith(".xml")
if is_valid:
return xml_file
else:
log_message("The provided file is not a valid XML file", level=logging.ERROR)
raise RuntimeError("The provided file is not a valid XML file")
def run_scan_process(self):
"""Function scans the source system"""
self.xml_mgr = xml_manager.XmlManager(self.conf.assessment_results_dir,
self.module_set_copy_path)
self.report_parser.add_global_tags(
self.conf.assessment_results_dir,
self.rename_custom_module_set(self.module_set_dirname),
self.conf.mode, self._devel_mode, self._dist_mode)
self.report_parser.modify_result_path(
self.conf.assessment_results_dir,
self.rename_custom_module_set(self.module_set_dirname),
self.conf.mode)
# Execute assessment
self.scanning_progress = ScanProgress(self.get_total_check(),
self.conf.debug)
self.scanning_progress.set_names(
self.report_parser.get_name_of_checks())
log_message('%s:' % settings.assessment_text, new_line=True)
log_message('%.3d/%.3d ...running (%s)' %
(1, self.get_total_check(),
self.scanning_progress.get_full_name(0)),
new_line=False)
start_time = datetime.datetime.now()
self.scanning_progress.time = start_time
self.run_scan(function=self.scanning_progress.show_progress)
end_time = datetime.datetime.now()
diff = end_time - start_time
log_message("The assessment finished (time %.2d:%.2ds)" %
(diff.seconds / 60, diff.seconds % 60))
def is_installed_oscap_ok():
"""Check whether expected openscap rpms are installed."""
class GetCmdStdout():
def __init__(self):
self.stdout_lines = []
def __call__(self, line):
if line.strip():
self.stdout_lines.append(line.strip())
if not os.path.exists(settings.openscap_binary):
log_message("Oscap with SCE enabled is not installed")
return False
if not os.access(settings.openscap_binary, os.X_OK):
log_message("Oscap with SCE %s is not executable"
% settings.openscap_binary)
return False
# that's generic problem that could be on various rpm-based systems
url = "https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html-single/6.10_release_notes/index#BZ1804691"
for pkg in settings.openscap_rpms:
cmd = ["rpm", "-q", pkg, "--qf", "%{ARCH}\n"]
cmdout = GetCmdStdout()
ProcessHelper.run_subprocess(cmd, function=cmdout)
if SystemIdentification.get_arch() not in cmdout.stdout_lines:
log_message("The %s rpm is not installed for the"
" %s architecture. This usually ends in a broken"
" state in which all the Preupgrade Assistant modules"
" are skipped (notchecked state). Please, install"
" packages related for your architecture. See %s"
" for more info."
% (pkg, SystemIdentification.get_arch(), url))
return False
return True
def embed_script(self, tarball):
if tarball is None:
return
tarball_content = None
if os.path.exists(tarball):
tarball_content = FileHelper.get_file_content(tarball, 'rb', decode_flag=False)
tarball_name = os.path.splitext(os.path.splitext(os.path.basename(tarball))[0])[0]
script_str = ''
try:
script_path = settings.KS_POSTSCRIPT_TEMPLATE
except AttributeError:
log_message('KS_POSTSCRIPT_TEMPLATE is not defined in settings.py.')
return
script_str = FileHelper.get_file_content(os.path.join(settings.KS_DIR, script_path), 'rb')
if not script_str:
log_message("Cannot open the script template: {0}.".format(script_path))
return
if tarball_content is not None:
script_str = script_str.replace('{tar_ball}', base64.b64encode(tarball_content))
script_str = script_str.replace('{RESULT_NAME}', tarball_name)
script_str = script_str.replace('{TEMPORARY_PREUPG_DIR}', '/root/preupgrade')
script = Script(script_str, type=KS_SCRIPT_POST, inChroot=True)
self.ks.handler.scripts.append(script)
def replace_inplace_risk(self, scanning_results=None):
"""
This function has aim to replace FAILED to
NEEDS_INSPECTION in case that risks are SLIGHT or MEDIUM
"""
#Filter all rule-result in TestResult
changed_fields = []
self.remove_empty_check_import()
inplace_dict = {
0: ReportHelper.upd_inspection,
1: ReportHelper.upd_action,
2: ReportHelper.upd_extreme,
}
for rule in self.get_all_result_rules():
result = [x for x in self.get_nodes(rule, "result") if x.text == "fail"]
# Get all affected rules and taken their names
for res in result:
inplace_risk = XccdfHelper.get_check_import_inplace_risk(rule)
logger_report.debug(inplace_risk)
# In case that report has state fail and
# no log_risk than it should be needs_inspection
if not inplace_risk:
changed = rule.get("idref")
res.text = "fail"
log_message('The %s module exits as fail but without a risk.' % rule.get("idref"))
else:
inplace_num = XccdfHelper.get_and_print_inplace_risk(0, inplace_risk)
logger_report.debug("Call function '%s'", inplace_dict[inplace_num])
changed, res.text = inplace_dict[inplace_num](rule)
logger_report.debug("Replace text '%s:%s'", changed, res.text)
if changed is not None:
changed_fields.append(changed+":"+res.text)
if scanning_results:
scanning_results.update_data(changed_fields)
self.write_xml()
def get_partition_layout(self, lsblk, vgs, lvdisplay):
"""
Returns dictionary with partition and realname and size
:param filename: filename with partition_layout in /root/preupgrade/kickstart directory
:return: dictionary with layout
"""
lsblk_filename = os.path.join(settings.KS_DIR, lsblk)
try:
self.layout = FileHelper.get_file_content(lsblk_filename,
'rb',
method=True,
decode_flag=False)
except IOError:
log_message("The %s file was not generated by the module. "
"Kickstart does not contain the partitioning layout" %
lsblk_filename)
self.part_layout = None
return None
if vgs is not None:
self.vg_info = PartitionGenerator.get_volume_info(
os.path.join(settings.KS_DIR, vgs), 0, 5)
if lvdisplay is not None:
self.lvdisplay = PartitionGenerator.get_volume_info(
os.path.join(settings.KS_DIR, lvdisplay), 0, 1)
def get_default_module_set_dirname(self):
available_module_set_dirs = get_installed_module_sets(
self.conf.source_dir)
if not available_module_set_dirs:
log_message("No modules found in the default directory (%s).\n"
" Either install a package with modules or use"
" -c option for custom created modules." %
settings.source_dir)
return None
if len(available_module_set_dirs) > 1:
log_message("More than one module set is detected in the default"
" directory (%s)." % settings.source_dir)
log_message("Available module sets: \n%s" %
'\n'.join(available_module_set_dirs))
log_message("Use option -s to specify which module set should be"
" used.")
return None
return available_module_set_dirs.keys()[0]
def get_default_module_set_dirname(self):
available_module_set_dirs = get_installed_module_sets(
self.conf.source_dir)
if not available_module_set_dirs:
log_message("No modules found in the default directory (%s).\n"
" Either install a package with modules or use"
" -c option for custom created modules."
% settings.source_dir)
return None
if len(available_module_set_dirs) > 1:
log_message("More than one module set is detected in the default"
" directory (%s)." % settings.source_dir)
log_message("Available module sets: \n%s"
% '\n'.join(available_module_set_dirs))
log_message("Use option -s to specify which module set should be"
" used.")
return None
return available_module_set_dirs.keys()[0]
def load_or_default(system_ks_path, ks_template):
""" load system ks or default ks """
ksparser = KickstartParser(makeVersion())
try:
ksparser.readKickstart(system_ks_path)
except (KickstartError, IOError):
log_message("Cannot read the system Kickstart at %s." % system_ks_path)
try:
ksparser.readKickstart(ks_template)
except AttributeError:
log_message("There is no KS_POSTSCRIPT_TEMPLATE specified in settings.py.", level=logging.DEBUG)
except IOError:
log_message("Cannot read the Kickstart template %s." % ks_template)
return None
return ksparser
def is_module_set_valid(self):
if self.module_set_dirname is None:
log_message('Invalid scenario: %s' % self.module_set_path)
return False
if not os.path.isdir(self.module_set_path):
log_message('Invalid scenario: %s' % self.module_set_path,
level=logging.ERROR)
return False
# Validate properties.ini file in module set dir
try:
ModuleSetUtils.get_module_set_os_versions(self.module_set_path)
except EnvironmentError as err:
log_message(str(err), level=logging.ERROR)
return False
return True