def run_jobs(pcs, job_matrix, number_of_jobs):
"""
Arguments:
pcs(PCS): object with performance control system wrapper
job_matrix(dict): dictionary with jobs that will be run
number_of_jobs(int): number of jobs that will be run
"""
for job_cmd, workloads_per_cmd in job_matrix.items():
log.print_current_phase("Collecting profiles for {}", job_cmd,
COLLECT_PHASE_CMD)
for workload, jobs_per_workload in workloads_per_cmd.items():
log.print_current_phase(" - processing workload {}", workload,
COLLECT_PHASE_WORKLOAD)
for job in jobs_per_workload:
log.print_job_progress(number_of_jobs)
# Run the collector and check if the profile was successfully collected
# In case, the status was not OK, then we skip the postprocessing
c_status, prof = run_collector(job.collector, job)
if c_status != CollectStatus.OK or not prof:
continue
for postprocessor in job.postprocessors:
log.print_job_progress(number_of_jobs)
# Run the postprocessor and check if the profile was successfully postprocessed
p_status, prof = run_postprocessor(postprocessor, job,
prof)
if p_status != PostprocessStatus.OK or not prof:
continue
# Store the computed profile inside the job directory
store_generated_profile(pcs, prof, job)
def run_collector(collector, job):
"""Run the job of collector of the given name.
Tries to look up the module containing the collector specified by the
collector name, and then runs it with the parameters and returns collected profile.
Arguments:
collector(Unit): object representing the collector
job(Job): additional information about the running job
Returns:
(int, dict): status of the collection, generated profile
"""
log.print_current_phase("Collecting data by {}", collector.name,
COLLECT_PHASE_COLLECT)
try:
collector_module = get_module('perun.collect.{0}.run'.format(
collector.name))
except ImportError:
return CollectStatus.ERROR, "{} does not exist".format(
collector.name), {}
# First init the collector by running the before phases (if it has)
job_params = utils.merge_dictionaries(job._asdict(), collector.params)
collection_status, collection_msg, prof \
= run_all_phases_for(collector_module, 'collector', job_params)
if collection_status != CollectStatus.OK:
log.error(collection_msg, recoverable=True)
else:
print("Successfully collected data from {}".format(job.cmd))
return collection_status, prof
def run_postprocessor(postprocessor, job, prof):
"""Run the job of postprocess of the given name.
Tries to look up the module containing the postprocessor specified by the
postprocessor name, and then runs it with the parameters and returns processed
profile.
:param Unit postprocessor: dictionary representing the postprocessor
:param Job job: additional information about the running job
:param dict prof: dictionary with profile
:returns (int, dict): status of the collection, postprocessed profile
"""
log.print_current_phase("Postprocessing data with {}", postprocessor.name,
COLLECT_PHASE_POSTPROCESS)
try:
postprocessor_module = get_module('perun.postprocess.{0}.run'.format(
postprocessor.name))
except ImportError:
err_msg = "{} does not exist".format(postprocessor.name)
log.error(err_msg, recoverable=True)
return PostprocessStatus.ERROR, {}
# First init the collector by running the before phases (if it has)
job_params = utils.merge_dict_range(job._asdict(), {'profile': prof},
postprocessor.params)
post_status, post_msg, prof \
= run_all_phases_for(postprocessor_module, 'postprocessor', job_params)
if post_status != PostprocessStatus.OK:
log.error(post_msg)
print("Successfully postprocessed data by {}".format(postprocessor.name))
return post_status, prof
def generate_jobs_on_current_working_dir(job_matrix, number_of_jobs):
"""Runs the batch of jobs on current state of the VCS.
This function expects no changes not commited in the repo, it excepts correct version
checked out and just runs the matrix.
:param dict job_matrix: dictionary with jobs that will be run
:param int number_of_jobs: number of jobs that will be run
:return: pair of job and generated profile
"""
workload_generators_specs = workloads.load_generator_specifications()
log.print_job_progress.current_job = 1
print("")
for job_cmd, workloads_per_cmd in job_matrix.items():
log.print_current_phase("Collecting profiles for {}", job_cmd,
COLLECT_PHASE_CMD)
for workload, jobs_per_workload in workloads_per_cmd.items():
log.print_current_phase(" = processing generator {}", workload,
COLLECT_PHASE_WORKLOAD)
# Prepare the specification
generator, params = workload_generators_specs.get(
workload, GeneratorSpec(SingletonGenerator,
{'value': workload}))
for job in jobs_per_workload:
log.print_job_progress(number_of_jobs)
for c_status, prof in generator(
job, **params).generate(run_collector):
# Run the collector and check if the profile was successfully collected
# In case, the status was not OK, then we skip the postprocessing
if c_status != CollectStatus.OK or not prof:
continue
# Temporary nasty hack
prof = profile.finalize_profile_for_job(prof, job)
for postprocessor in job.postprocessors:
log.print_job_progress(number_of_jobs)
# Run postprocess and check if the profile was successfully postprocessed
p_status, prof = run_postprocessor(
postprocessor, job, prof)
if p_status != PostprocessStatus.OK or not prof:
break
else:
# Store the computed profile inside the job directory
yield prof, job