def run_collector(collector, job):
"""Run the job of collector of the given name.
Tries to look up the module containing the collector specified by the
collector name, and then runs it with the parameters and returns collected profile.
Arguments:
collector(Unit): object representing the collector
job(Job): additional information about the running job
Returns:
(int, dict): status of the collection, generated profile
"""
log.print_current_phase("Collecting data by {}", collector.name,
COLLECT_PHASE_COLLECT)
try:
collector_module = get_module('perun.collect.{0}.run'.format(
collector.name))
except ImportError:
return CollectStatus.ERROR, "{} does not exist".format(
collector.name), {}
# First init the collector by running the before phases (if it has)
job_params = utils.merge_dictionaries(job._asdict(), collector.params)
collection_status, collection_msg, prof \
= run_all_phases_for(collector_module, 'collector', job_params)
if collection_status != CollectStatus.OK:
log.error(collection_msg, recoverable=True)
else:
print("Successfully collected data from {}".format(job.cmd))
return collection_status, prof
def assert_all_registered_modules(package_name, package,
must_have_function_names):
"""Asserts that for given package all of its modules are properly registered in Perun
Moreover checks that all of the must have functions are implemented as well.
Arguments:
package_name(str): name of the package we are checking all the modules
package(module): checked package
must_have_function_names(list): list of functions that the module from package has to have
registered
"""
registered_modules = utils.get_supported_module_names(package_name)
for (_, module_name, _) in pkgutil.iter_modules(package.__path__,
package.__name__ + '.'):
module = utils.get_module(module_name)
for must_have_function_name in must_have_function_names:
assert hasattr(
module,
must_have_function_name) and "Missing {} in module {}".format(
must_have_function_name, module_name)
# Each module has to be registered in get_supported_module_names
unit_name = module_name.split('.')[-1]
assert unit_name in registered_modules and "{} was not registered properly".format(
module_name)
def run_postprocessor(postprocessor, job, prof):
"""Run the job of postprocess of the given name.
Tries to look up the module containing the postprocessor specified by the
postprocessor name, and then runs it with the parameters and returns processed
profile.
:param Unit postprocessor: dictionary representing the postprocessor
:param Job job: additional information about the running job
:param dict prof: dictionary with profile
:returns (int, dict): status of the collection, postprocessed profile
"""
log.print_current_phase("Postprocessing data with {}", postprocessor.name,
COLLECT_PHASE_POSTPROCESS)
try:
postprocessor_module = get_module('perun.postprocess.{0}.run'.format(
postprocessor.name))
except ImportError:
err_msg = "{} does not exist".format(postprocessor.name)
log.error(err_msg, recoverable=True)
return PostprocessStatus.ERROR, {}
# First init the collector by running the before phases (if it has)
job_params = utils.merge_dict_range(job._asdict(), {'profile': prof},
postprocessor.params)
post_status, post_msg, prof \
= run_all_phases_for(postprocessor_module, 'postprocessor', job_params)
if post_status != PostprocessStatus.OK:
log.error(post_msg)
print("Successfully postprocessed data by {}".format(postprocessor.name))
return post_status, prof
def assert_all_registered_cli_units(package_name, package,
must_have_function_names):
"""Asserts that for given package all of its modules are properly registered in Perun
Moreover checks that it has the CLI interface function in order to be called through click,
and that certain functions are implemented as well (namely collect/postprocess) in order
to automate the process of profile generation and postprocessing.
Arguments:
package_name(str): name of the package we are checking all the modules
package(module): checked package (one of collect, postprocess, view)
must_have_function_names(list): list of functions that the module from package has to have
registered
"""
registered_modules = utils.get_supported_module_names(package_name)
for (_, module_name, _) in pkgutil.iter_modules(package.__path__,
package.__name__ + '.'):
# Each module has to have run.py module
module = utils.get_module(module_name)
assert hasattr(
module, 'run'
) and "Missing module run.py in the '{}' module".format(package_name)
run_module = utils.get_module(".".join([module_name, "run"]))
for must_have_function_name in must_have_function_names:
assert not must_have_function_name or hasattr(run_module, must_have_function_name) and \
"run.py is missing '{}' function".format(must_have_function_name)
# Each module has to have CLI interface function of the same name
unit_name = module_name.split('.')[-1]
assert hasattr(
run_module,
unit_name) and "{} is missing CLI function point".format(unit_name)
# Each module has to be registered in get_supported_module_names
assert unit_name in registered_modules and "{} was not registered properly".format(
module_name)
def load_generator_specifications():
"""Collects from configuration file all of the workload specifications and constructs a mapping
of 'id' -> GeneratorSpec, which contains the constructor and parameters used for construction.
The specifications are specified by :ckey:`generators.workload` as follows:
generators:
workload:
- id: name_of_generator
type: integer
min_range: 10
max_range: 20
:return: map of ids to generator specifications
"""
specifications_from_config = config.gather_key_recursively(
'generators.workload')
spec_map = {}
warnings = []
print("Loading workload generator specifications ", end='')
for spec in specifications_from_config:
if 'id' not in spec.keys() or 'type' not in spec.keys():
warnings.append(
"incorrect workload specification: missing 'id' or 'type'")
print("F", end='')
continue
generator_module = "perun.workload.{}_generator".format(
spec['type'].lower())
constructor_name = "{}Generator".format(spec['type'].title())
try:
constructor = getattr(utils.get_module(generator_module),
constructor_name)
spec_map[spec['id']] = GeneratorSpec(constructor, spec)
print(".", end='')
except (ImportError, AttributeError):
warnings.append(
"incorrect workload generator '{}': '{}' is not valid workload type"
.format(spec['id'], spec['type']))
print("F", end='')
# Print the warnings and badge
if len(warnings):
log.failed()
print("\n".join(warnings))
else:
log.done()
return spec_map
def generate_header_for_profile(job):
"""
:param Job job: job with information about the computed profile
:returns dict: dictionary in form of {'header': {}} corresponding to the perun specification
"""
try:
collector = get_module('.'.join(['perun.collect', job.collector.name]))
except ImportError:
perun_log.error("could not find package for collector {}".format(job.collector.name))
return {
'type': collector.COLLECTOR_TYPE,
'cmd': job.cmd,
'params': job.args,
'workload': job.workload,
'units': generate_units(collector)
}