def archive_logfiles():
""" Archive log files from a previous run of Leapp """
cfg = get_config()
if not os.path.isdir(cfg.get('logs', 'dir')):
os.makedirs(cfg.get('logs', 'dir'))
files_to_archive = [os.path.join(cfg.get('logs', 'dir'), f)
for f in cfg.get('logs', 'files').split(',')
if os.path.isfile(os.path.join(cfg.get('logs', 'dir'), f))]
if not os.path.isdir(cfg.get('archive', 'dir')):
os.makedirs(cfg.get('archive', 'dir'))
if files_to_archive:
if os.path.isdir(cfg.get('debug', 'dir')):
files_to_archive.append(cfg.get('debug', 'dir'))
now = datetime.now().strftime('%Y%m%d%H%M%S')
archive_file = os.path.join(cfg.get('archive', 'dir'), 'leapp-{}-logs.tar.gz'.format(now))
with tarfile.open(archive_file, "w:gz") as tar:
for file_to_add in files_to_archive:
tar.add(file_to_add)
if os.path.isdir(file_to_add):
shutil.rmtree(file_to_add, ignore_errors=True)
try:
os.remove(file_to_add)
except OSError:
pass
# leapp_db is not in files_to_archive to not have it removed
if os.path.isfile(cfg.get('database', 'path')):
tar.add(cfg.get('database', 'path'))
def answer(args):
"""A command to record user choices to the questions in the answerfile.
Saves user answer between leapp preupgrade runs.
"""
cfg = get_config()
if args.section:
args.section = list(
itertools.chain(*[i.split(',') for i in args.section]))
else:
raise UsageError(
'At least one dialog section must be specified, ex. --section dialog.option=mychoice'
)
try:
sections = [
tuple((dialog_option.split('.', 2) + [value])) for dialog_option,
value in [s.split('=', 2) for s in args.section]
]
except ValueError:
raise UsageError(
"A bad formatted section has been passed. Expected format is dialog.option=mychoice"
)
answerfile_path = cfg.get('report', 'answerfile')
answerstore = AnswerStore()
answerstore.load(answerfile_path)
for dialog, option, value in sections:
answerstore.answer(dialog, option, value)
not_updated = answerstore.update(answerfile_path, allow_missing=args.add)
if not_updated:
sys.stderr.write(
"WARNING: Only sections found in original userfile can be updated, ignoring {}\n"
.format(",".join(not_updated)))
def test_migrations_are_applied():
con = sqlite3.connect(get_config().get('database', 'path'))
con.executescript(_ORIGINAL_DB_SCHEMA)
assert con.execute("PRAGMA user_version").fetchone()[0] == 0
con.close()
with get_connection(None) as db:
assert db.execute("PRAGMA user_version").fetchone()[0] > 0
def upgrade(args):
skip_phases_until = None
context = str(uuid.uuid4())
cfg = get_config()
configuration = prepare_configuration(args)
answerfile_path = cfg.get('report', 'answerfile')
userchoices_path = cfg.get('report', 'userchoices')
if os.getuid():
raise CommandError('This command has to be run under the root user.')
handle_output_level(args)
if args.resume:
context, configuration = fetch_last_upgrade_context()
if not context:
raise CommandError('No previous upgrade run to continue, remove `--resume` from leapp invocation to'
'start a new upgrade flow')
os.environ['LEAPP_DEBUG'] = '1' if check_env_and_conf('LEAPP_DEBUG', 'debug', configuration) else '0'
if os.environ['LEAPP_DEBUG'] == '1' or check_env_and_conf('LEAPP_VERBOSE', 'verbose', configuration):
os.environ['LEAPP_VERBOSE'] = '1'
else:
os.environ['LEAPP_VERBOSE'] = '0'
skip_phases_until = get_last_phase(context)
logger = configure_logger()
else:
e = Execution(context=context, kind='upgrade', configuration=configuration)
e.store()
archive_logfiles()
logger = configure_logger('leapp-upgrade.log')
os.environ['LEAPP_EXECUTION_ID'] = context
if args.resume:
logger.info("Resuming execution after phase: %s", skip_phases_until)
try:
repositories = load_repositories()
except LeappError as exc:
raise CommandError(exc.message)
workflow = repositories.lookup_workflow('IPUWorkflow')(auto_reboot=args.reboot)
process_whitelist_experimental(repositories, workflow, configuration, logger)
warn_if_unsupported(configuration)
with beautify_actor_exception():
logger.info("Using answerfile at %s", answerfile_path)
workflow.load_answers(answerfile_path, userchoices_path)
workflow.run(context=context, skip_phases_until=skip_phases_until, skip_dialogs=True)
logger.info("Answerfile will be created at %s", answerfile_path)
workflow.save_answers(answerfile_path, userchoices_path)
report_errors(workflow.errors)
report_inhibitors(context)
generate_report_files(context)
report_files = get_cfg_files('report', cfg)
log_files = get_cfg_files('logs', cfg)
report_info(report_files, log_files, answerfile_path, fail=workflow.failure)
if workflow.failure:
sys.exit(1)
def preupgrade(args):
if os.getuid():
raise CommandError('This command has to be run under the root user.')
if args.whitelist_experimental:
args.whitelist_experimental = list(
itertools.chain(
*[i.split(',') for i in args.whitelist_experimental]))
context = str(uuid.uuid4())
configuration = {
'debug': os.getenv('LEAPP_DEBUG', '0'),
'verbose': os.getenv('LEAPP_VERBOSE', '0'),
'whitelist_experimental': args.whitelist_experimental or ()
}
e = Execution(context=context,
kind='preupgrade',
configuration=configuration)
e.store()
archive_logfiles()
logger = configure_logger('leapp-preupgrade.log')
os.environ['LEAPP_EXECUTION_ID'] = context
try:
repositories = load_repositories()
except LeappError as exc:
raise CommandError(exc.message)
workflow = repositories.lookup_workflow('IPUWorkflow')()
for actor_name in configuration.get('whitelist_experimental', ()):
actor = repositories.lookup_actor(actor_name)
if actor:
workflow.whitelist_experimental_actor(actor)
else:
msg = 'No such Actor: {}'.format(actor_name)
logger.error(msg)
raise CommandError(msg)
with beautify_actor_exception():
until_phase = 'ReportsPhase'
logger.info('Executing workflow until phase: %s', until_phase)
workflow.run(context=context,
until_phase=until_phase,
skip_dialogs=True)
cfg = get_config()
answerfile = args.save_answerfile or cfg.get('report', 'answerfile')
logger.info("Answerfile will be created at %s", answerfile)
workflow._answer_store.generate_for_workflow(workflow, answerfile)
generate_report_files(context)
report_errors(workflow.errors)
report_files = [
os.path.join(cfg.get('report', 'dir'), r)
for r in cfg.get('report', 'files').split(',')
]
report_info([rf for rf in report_files if os.path.isfile(rf)],
fail=workflow.failure)
if workflow.failure:
sys.exit(1)
def generate_report_files(context):
"""
Generates all report files for specific leapp run (txt and json format)
"""
cfg = get_config()
report_txt, report_json = [os.path.join(cfg.get('report', 'dir'),
'leapp-report.{}'.format(f)) for f in ['txt', 'json']]
# fetch all report messages as a list of dicts
messages = fetch_upgrade_report_messages(context)
generate_report_file(messages, context, report_json)
generate_report_file(messages, context, report_txt)
def get_connection(db):
"""
Get the database connection or passes it through if it is already set
:param db: Database connection to be passed through in case it exists already
:return: database object initialized and migrated to the latest schema version
"""
if db:
return db
cfg = get_config()
return create_connection(cfg.get('database', 'path'))
def preupgrade(args, breadcrumbs):
context = str(uuid.uuid4())
cfg = get_config()
util.handle_output_level(args)
configuration = util.prepare_configuration(args)
answerfile_path = cfg.get('report', 'answerfile')
userchoices_path = cfg.get('report', 'userchoices')
if os.getuid():
raise CommandError('This command has to be run under the root user.')
e = Execution(context=context,
kind='preupgrade',
configuration=configuration)
e.store()
util.archive_logfiles()
logger = configure_logger('leapp-preupgrade.log')
os.environ['LEAPP_EXECUTION_ID'] = context
try:
repositories = util.load_repositories()
except LeappError as exc:
raise CommandError(exc.message)
workflow = repositories.lookup_workflow('IPUWorkflow')()
util.warn_if_unsupported(configuration)
util.process_whitelist_experimental(repositories, workflow, configuration,
logger)
with beautify_actor_exception():
workflow.load_answers(answerfile_path, userchoices_path)
until_phase = 'ReportsPhase'
logger.info('Executing workflow until phase: %s', until_phase)
# Set the locale, so that the actors parsing command outputs that might be localized will not fail
os.environ['LC_ALL'] = 'en_US.UTF-8'
os.environ['LANG'] = 'en_US.UTF-8'
workflow.run(context=context,
until_phase=until_phase,
skip_dialogs=True)
logger.info("Answerfile will be created at %s", answerfile_path)
workflow.save_answers(answerfile_path, userchoices_path)
util.generate_report_files(context)
report_errors(workflow.errors)
report_inhibitors(context)
report_files = util.get_cfg_files('report', cfg)
log_files = util.get_cfg_files('logs', cfg)
report_info(report_files,
log_files,
answerfile_path,
fail=workflow.failure)
if workflow.failure:
sys.exit(1)
def preupgrade(args):
if os.getuid():
raise CommandError('This command has to be run under the root user.')
if args.whitelist_experimental:
args.whitelist_experimental = list(itertools.chain(*[i.split(',') for i in args.whitelist_experimental]))
context = str(uuid.uuid4())
configuration = {
'debug': os.getenv('LEAPP_DEBUG', '0'),
'verbose': os.getenv('LEAPP_VERBOSE', '0'),
'whitelist_experimental': args.whitelist_experimental or ()
}
e = Execution(context=context, kind='preupgrade', configuration=configuration)
e.store()
archive_logfiles()
logger = configure_logger('leapp-preupgrade.log')
os.environ['LEAPP_EXECUTION_ID'] = context
try:
repositories = load_repositories()
except LeappError as exc:
raise CommandError(exc.message)
workflow = repositories.lookup_workflow('IPUWorkflow')()
for actor_name in configuration.get('whitelist_experimental', ()):
actor = repositories.lookup_actor(actor_name)
if actor:
workflow.whitelist_experimental_actor(actor)
else:
msg = 'No such Actor: {}'.format(actor_name)
logger.error(msg)
raise CommandError(msg)
with beautify_actor_exception():
until_phase = 'ReportsPhase'
logger.info('Executing workflow until phase: %s', until_phase)
workflow.run(context=context, until_phase=until_phase)
report_errors(workflow.errors)
report_txt, report_json = [os.path.join(get_config().get('report', 'dir'),
'leapp-report.{}'.format(f)) for f in ['txt', 'json']]
report_info([report_txt, report_json], fail=workflow.errors)
# fetch all report messages as a list of dicts
messages = fetch_upgrade_report_raw(context, renderers=False)
with open(report_json, 'w+') as f:
json.dump({'entries': messages}, f, indent=2)
if workflow.failure:
sys.exit(1)
def configure_logger(log_file=None):
global _logger
if not _logger:
log_format = '%(asctime)s.%(msecs)-3d %(levelname)-8s PID: %(process)d %(name)s: %(message)s'
log_date_format = '%Y-%m-%d %H:%M:%S'
path = os.getenv('LEAPP_LOGGER_CONFIG', '/etc/leapp/logger.conf')
if path and os.path.isfile(path):
logging.config.fileConfig(path)
else: # Fall back logging configuration
logging.Formatter.converter = time.gmtime
logging.basicConfig(
level=logging.ERROR,
format=log_format,
datefmt=log_date_format,
stream=sys.stderr,
)
logging.getLogger('urllib3').setLevel(logging.WARN)
handler = LeappAuditHandler()
handler.setFormatter(
logging.Formatter(fmt=log_format, datefmt=log_date_format))
logging.getLogger('leapp').addHandler(handler)
if log_file:
file_handler = logging.FileHandler(
os.path.join(get_config().get('logs', 'dir'), log_file))
file_handler.setFormatter(
logging.Formatter(fmt=log_format, datefmt=log_date_format))
file_handler.setLevel(logging.DEBUG)
logging.getLogger('leapp').addHandler(file_handler)
if is_verbose():
for handler in logging.getLogger().handlers:
if isinstance(handler, logging.StreamHandler):
handler.setLevel(
logging.DEBUG if is_debug() else logging.INFO)
_logger = logging.getLogger('leapp')
_logger.info('Logging has been initialized')
return _logger
def setup():
path = get_config().get('database', 'path')
if os.path.isfile(path):
os.unlink(path)
def setup_module():
get_config().set('database', 'path', '/tmp/leapp-test.db')
def load_repositories_from(name, repo_path, manager=None):
if get_config().has_option('repositories', name):
repo_path = get_config().get('repositories', name)
return find_and_scan_repositories(repo_path, manager=manager)
def upgrade(args, breadcrumbs):
skip_phases_until = None
context = str(uuid.uuid4())
cfg = get_config()
util.handle_output_level(args)
answerfile_path = cfg.get('report', 'answerfile')
userchoices_path = cfg.get('report', 'userchoices')
# Processing of parameters passed by the rerun call, these aren't actually command line arguments
# therefore we have to assume that they aren't even in `args` as they are added only by rerun.
only_with_tags = args.only_with_tags if 'only_with_tags' in args else None
resume_context = args.resume_context if 'resume_context' in args else None
report_schema = util.process_report_schema(args, cfg)
if os.getuid():
raise CommandError('This command has to be run under the root user.')
if args.resume:
context, configuration = util.fetch_last_upgrade_context(resume_context)
if not context:
raise CommandError('No previous upgrade run to continue, remove `--resume` from leapp invocation to'
' start a new upgrade flow')
os.environ['LEAPP_DEBUG'] = '1' if util.check_env_and_conf('LEAPP_DEBUG', 'debug', configuration) else '0'
if os.environ['LEAPP_DEBUG'] == '1' or util.check_env_and_conf('LEAPP_VERBOSE', 'verbose', configuration):
os.environ['LEAPP_VERBOSE'] = '1'
else:
os.environ['LEAPP_VERBOSE'] = '0'
util.restore_leapp_env_vars(context)
skip_phases_until = util.get_last_phase(context)
else:
util.disable_database_sync()
configuration = util.prepare_configuration(args)
e = Execution(context=context, kind='upgrade', configuration=configuration)
e.store()
util.archive_logfiles()
logger = configure_logger('leapp-upgrade.log')
os.environ['LEAPP_EXECUTION_ID'] = context
if args.resume:
logger.info("Resuming execution after phase: %s", skip_phases_until)
try:
repositories = util.load_repositories()
except LeappError as exc:
raise CommandError(exc.message)
workflow = repositories.lookup_workflow('IPUWorkflow')(auto_reboot=args.reboot)
util.process_whitelist_experimental(repositories, workflow, configuration, logger)
util.warn_if_unsupported(configuration)
with beautify_actor_exception():
logger.info("Using answerfile at %s", answerfile_path)
workflow.load_answers(answerfile_path, userchoices_path)
# Set the locale, so that the actors parsing command outputs that might be localized will not fail
os.environ['LC_ALL'] = 'en_US.UTF-8'
os.environ['LANG'] = 'en_US.UTF-8'
workflow.run(context=context, skip_phases_until=skip_phases_until, skip_dialogs=True,
only_with_tags=only_with_tags)
logger.info("Answerfile will be created at %s", answerfile_path)
workflow.save_answers(answerfile_path, userchoices_path)
report_errors(workflow.errors)
report_inhibitors(context)
util.generate_report_files(context, report_schema)
report_files = util.get_cfg_files('report', cfg)
log_files = util.get_cfg_files('logs', cfg)
report_info(report_files, log_files, answerfile_path, fail=workflow.failure)
if workflow.failure:
sys.exit(1)
@command_opt('whitelist-experimental', action='append', metavar='ActorName', help='Enable experimental actors')
@command_opt('debug', is_flag=True, help='Enable debug mode', inherit=False)
@command_opt('verbose', is_flag=True, help='Enable verbose logging', inherit=False)
@command_opt('no-rhsm', is_flag=True, help='Use only custom repositories and skip actions'
' with Red Hat Subscription Manager')
@command_opt('enablerepo', action='append', metavar='<repoid>',
help='Enable specified repository. Can be used multiple times.')
@command_opt('channel',
help='Set preferred channel for the IPU target.',
choices=['ga', 'tuv', 'e4s', 'eus', 'aus'],
value_type=str.lower) # This allows the choices to be case insensitive
@command_opt('target', choices=command_utils.get_supported_target_versions(),
help='Specify RHEL version to upgrade to for {} detected upgrade flavour'.format(
command_utils.get_upgrade_flavour()))
@command_opt('report-schema', help='Specify report schema version for leapp-report.json', choices=['1.0.0', '1.1.0'],
default=get_config().get('report', 'schema'))
@breadcrumbs.produces_breadcrumbs
def upgrade(args, breadcrumbs):
skip_phases_until = None
context = str(uuid.uuid4())
cfg = get_config()
util.handle_output_level(args)
answerfile_path = cfg.get('report', 'answerfile')
userchoices_path = cfg.get('report', 'userchoices')
# Processing of parameters passed by the rerun call, these aren't actually command line arguments
# therefore we have to assume that they aren't even in `args` as they are added only by rerun.
only_with_tags = args.only_with_tags if 'only_with_tags' in args else None
resume_context = args.resume_context if 'resume_context' in args else None
report_schema = util.process_report_schema(args, cfg)
def upgrade(args):
if os.getuid():
raise CommandError('This command has to be run under the root user.')
if args.whitelist_experimental:
args.whitelist_experimental = list(
itertools.chain(
*[i.split(',') for i in args.whitelist_experimental]))
skip_phases_until = None
context = str(uuid.uuid4())
configuration = {
'debug': os.getenv('LEAPP_DEBUG', '0'),
'verbose': os.getenv('LEAPP_VERBOSE', '0'),
'whitelist_experimental': args.whitelist_experimental or ()
}
if args.resume:
context, configuration = fetch_last_upgrade_context()
if not context:
raise CommandError(
'No previous upgrade run to continue, remove `--resume` from leapp invocation to'
'start a new upgrade flow')
os.environ['LEAPP_DEBUG'] = '1' if check_env_and_conf(
'LEAPP_DEBUG', 'debug', configuration) else '0'
if os.environ['LEAPP_DEBUG'] == '1' or check_env_and_conf(
'LEAPP_VERBOSE', 'verbose', configuration):
os.environ['LEAPP_VERBOSE'] = '1'
else:
os.environ['LEAPP_VERBOSE'] = '0'
skip_phases_until = get_last_phase(context)
logger = configure_logger()
else:
e = Execution(context=context,
kind='upgrade',
configuration=configuration)
e.store()
archive_logfiles()
logger = configure_logger('leapp-upgrade.log')
os.environ['LEAPP_EXECUTION_ID'] = context
if args.resume:
logger.info("Resuming execution after phase: %s", skip_phases_until)
try:
repositories = load_repositories()
except LeappError as exc:
raise CommandError(exc.message)
workflow = repositories.lookup_workflow('IPUWorkflow')(
auto_reboot=args.reboot)
for actor_name in configuration.get('whitelist_experimental', ()):
actor = repositories.lookup_actor(actor_name)
if actor:
workflow.whitelist_experimental_actor(actor)
else:
msg = 'No such Actor: {}'.format(actor_name)
logger.error(msg)
raise CommandError(msg)
with beautify_actor_exception():
answerfile_path = args.load_answerfile or get_config().get(
'report', 'answerfile')
logger.info("Using answerfile at %s", answerfile_path)
workflow.load_answerfile(answerfile_path)
workflow.run(context=context, skip_phases_until=skip_phases_until)
report_errors(workflow.errors)
generate_report_files(context)
if workflow.failure:
sys.exit(1)