def collect_configuration_run_summary_logs(config_filenames, args):
'''
Given a sequence of configuration filenames and parsed command-line arguments as an
argparse.ArgumentParser instance, run each configuration file and yield a series of
logging.LogRecord instances containing summary information about each run.
'''
# Dict mapping from config filename to corresponding parsed config dict.
configs = collections.OrderedDict()
# Parse and load each configuration file.
for config_filename in config_filenames:
try:
logger.info('{}: Parsing configuration file'.format(config_filename))
configs[config_filename] = validate.parse_configuration(
config_filename, validate.schema_filename()
)
except (ValueError, OSError, validate.Validation_error) as error:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
msg='{}: Error parsing configuration file'.format(config_filename),
)
)
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
# Run cross-file validation checks.
if args.extract or (args.list and args.archive):
try:
validate.guard_configuration_contains_repository(args.repository, configs)
except ValueError as error:
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
return
# Execute the actions corresponding to each configuration file.
for config_filename, config in configs.items():
try:
run_configuration(config_filename, config, args)
yield logging.makeLogRecord(
dict(
levelno=logging.INFO,
msg='{}: Successfully ran configuration file'.format(config_filename),
)
)
except (ValueError, OSError, CalledProcessError) as error:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
msg='{}: Error running configuration file'.format(config_filename),
)
)
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
if not config_filenames:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
msg='{}: No configuration files found'.format(' '.join(args.config_paths)),
)
)
def test_guard_configuration_contains_repository_errors_when_repository_matches_config_twice():
with pytest.raises(ValueError):
module.guard_configuration_contains_repository(
repository='repo',
configurations={
'config.yaml': {'location': {'repositories': ['repo', 'repo2']}},
'other.yaml': {'location': {'repositories': ['repo']}},
},
)
def test_guard_configuration_contains_repository_does_not_raise_when_repository_not_given(
):
module.guard_configuration_contains_repository(repository=None,
configurations={
'config.yaml': {
'location': {
'repositories':
['repo']
}
}
})
def test_guard_configuration_contains_repository_errors_when_repository_missing_from_config(
):
with pytest.raises(ValueError):
module.guard_configuration_contains_repository(
repository='nope',
configurations={
'config.yaml': {
'location': {
'repositories': ['repo', 'repo2']
}
}
},
)
def test_guard_configuration_contains_repository_errors_when_repository_assumed_to_match_config_twice(
):
with pytest.raises(ValueError):
module.guard_configuration_contains_repository(
repository=None,
configurations={
'config.yaml': {
'location': {
'repositories': ['repo', 'repo2']
}
}
},
)
def test_guard_configuration_contains_repository_does_not_raise_when_repository_in_config(
):
flexmock(module).should_receive('repositories_match').replace_with(
lambda first, second: first == second)
module.guard_configuration_contains_repository(repository='repo',
configurations={
'config.yaml': {
'location': {
'repositories':
['repo']
}
}
})
def test_guard_configuration_contains_repository_errors_when_repository_missing_from_config(
):
flexmock(module).should_receive('repositories_match').replace_with(
lambda first, second: first == second)
with pytest.raises(ValueError):
module.guard_configuration_contains_repository(
repository='nope',
configurations={
'config.yaml': {
'location': {
'repositories': ['repo', 'repo2']
}
}
},
)
def collect_configuration_run_summary_logs(configs, arguments):
'''
Given a dict of configuration filename to corresponding parsed configuration, and parsed
command-line arguments as a dict from subparser name to a parsed namespace of arguments, run
each configuration file and yield a series of logging.LogRecord instances containing summary
information about each run.
As a side effect of running through these configuration files, output their JSON results, if
any, to stdout.
'''
# Run cross-file validation checks.
if 'extract' in arguments:
repository = arguments['extract'].repository
elif 'list' in arguments and arguments['list'].archive:
repository = arguments['list'].repository
else:
repository = None
if repository:
try:
validate.guard_configuration_contains_repository(
repository, configs)
except ValueError as error:
yield from make_error_log_records(str(error))
return
if not configs:
yield from make_error_log_records(
'{}: No configuration files found'.format(' '.join(
arguments['global'].config_paths)))
return
if 'create' in arguments:
try:
for config_filename, config in configs.items():
hooks = config.get('hooks', {})
command.execute_hook(
hooks.get('before_everything'),
hooks.get('umask'),
config_filename,
'pre-everything',
arguments['global'].dry_run,
)
except (CalledProcessError, ValueError, OSError) as error:
yield from make_error_log_records(
'Error running pre-everything hook', error)
return
# Execute the actions corresponding to each configuration file.
json_results = []
for config_filename, config in configs.items():
results = list(run_configuration(config_filename, config, arguments))
error_logs = tuple(result for result in results
if isinstance(result, logging.LogRecord))
if error_logs:
yield from make_error_log_records(
'{}: Error running configuration file'.format(config_filename))
yield from error_logs
else:
yield logging.makeLogRecord(
dict(
levelno=logging.INFO,
levelname='INFO',
msg='{}: Successfully ran configuration file'.format(
config_filename),
))
if results:
json_results.extend(results)
if json_results:
sys.stdout.write(json.dumps(json_results))
if 'create' in arguments:
try:
for config_filename, config in configs.items():
hooks = config.get('hooks', {})
command.execute_hook(
hooks.get('after_everything'),
hooks.get('umask'),
config_filename,
'post-everything',
arguments['global'].dry_run,
)
except (CalledProcessError, ValueError, OSError) as error:
yield from make_error_log_records(
'Error running post-everything hook', error)
def test_guard_configuration_contains_repository_does_not_raise_when_repository_in_config():
module.guard_configuration_contains_repository(
repository='repo', configurations={'config.yaml': {'location': {'repositories': ['repo']}}}
)
def test_guard_configuration_contains_repository_errors_when_repository_missing_from_config():
with pytest.raises(ValueError):
module.guard_configuration_contains_repository(
repository='nope',
configurations={'config.yaml': {'location': {'repositories': ['repo', 'repo2']}}},
)
def test_guard_configuration_contains_repository_errors_when_repository_assumed_to_match_config_twice():
with pytest.raises(ValueError):
module.guard_configuration_contains_repository(
repository=None,
configurations={'config.yaml': {'location': {'repositories': ['repo', 'repo2']}}},
)
def collect_configuration_run_summary_logs(config_filenames, args):
'''
Given a sequence of configuration filenames and parsed command-line arguments as an
argparse.ArgumentParser instance, run each configuration file and yield a series of
logging.LogRecord instances containing summary information about each run.
As a side effect of running through these configuration files, output their JSON results, if
any, to stdout.
'''
# Dict mapping from config filename to corresponding parsed config dict.
configs = collections.OrderedDict()
# Parse and load each configuration file.
for config_filename in config_filenames:
try:
logger.info('{}: Parsing configuration file'.format(config_filename))
configs[config_filename] = validate.parse_configuration(
config_filename, validate.schema_filename()
)
except (ValueError, OSError, validate.Validation_error) as error:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
msg='{}: Error parsing configuration file'.format(config_filename),
)
)
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
# Run cross-file validation checks.
if args.extract or (args.list and args.archive):
try:
validate.guard_configuration_contains_repository(args.repository, configs)
except ValueError as error:
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
return
# Execute the actions corresponding to each configuration file.
json_results = []
for config_filename, config in configs.items():
try:
json_results.extend(list(run_configuration(config_filename, config, args)))
yield logging.makeLogRecord(
dict(
levelno=logging.INFO,
msg='{}: Successfully ran configuration file'.format(config_filename),
)
)
except (ValueError, OSError, CalledProcessError) as error:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
msg='{}: Error running configuration file'.format(config_filename),
)
)
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
if json_results:
sys.stdout.write(json.dumps(json_results))
if not config_filenames:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
msg='{}: No configuration files found'.format(' '.join(args.config_paths)),
)
)
def collect_configuration_run_summary_logs(configs, arguments):
'''
Given a dict of configuration filename to corresponding parsed configuration, and parsed
command-line arguments as a dict from subparser name to a parsed namespace of arguments, run
each configuration file and yield a series of logging.LogRecord instances containing summary
information about each run.
As a side effect of running through these configuration files, output their JSON results, if
any, to stdout.
'''
# Run cross-file validation checks.
if 'extract' in arguments:
repository = arguments['extract'].repository
elif 'list' in arguments and arguments['list'].archive:
repository = arguments['list'].repository
else:
repository = None
if repository:
try:
validate.guard_configuration_contains_repository(
repository, configs)
except ValueError as error:
yield logging.makeLogRecord(
dict(levelno=logging.CRITICAL, levelname='CRITICAL',
msg=error))
return
# Execute the actions corresponding to each configuration file.
json_results = []
for config_filename, config in configs.items():
try:
json_results.extend(
list(run_configuration(config_filename, config, arguments)))
yield logging.makeLogRecord(
dict(
levelno=logging.INFO,
levelname='INFO',
msg='{}: Successfully ran configuration file'.format(
config_filename),
))
except CalledProcessError as error:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
levelname='CRITICAL',
msg='{}: Error running configuration file'.format(
config_filename),
))
yield logging.makeLogRecord(
dict(levelno=logging.CRITICAL,
levelname='CRITICAL',
msg=error.output))
yield logging.makeLogRecord(
dict(levelno=logging.CRITICAL, levelname='CRITICAL',
msg=error))
except (ValueError, OSError) as error:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
levelname='CRITICAL',
msg='{}: Error running configuration file'.format(
config_filename),
))
yield logging.makeLogRecord(
dict(levelno=logging.CRITICAL, levelname='CRITICAL',
msg=error))
if json_results:
sys.stdout.write(json.dumps(json_results))
if not configs:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
levelname='CRITICAL',
msg='{}: No configuration files found'.format(' '.join(
arguments['global'].config_paths)),
))