def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybuild-build', prepare_first=True):
"""
Build easyconfigs in parallel by submitting jobs to a batch-queuing system.
Return list of jobs submitted.
Argument `easyconfigs` is a list of easyconfigs which can be
built: e.g. they have no unresolved dependencies. This function
will build them in parallel by submitting jobs.
:param build_command: build command to use
:param easyconfigs: list of easyconfig files
:param output_dir: output directory
:param prepare_first: prepare by runnning fetch step first for each easyconfig
"""
_log.info("going to build these easyconfigs in parallel: %s", easyconfigs)
active_job_backend = job_backend()
if active_job_backend is None:
raise EasyBuildError("Can not use --job if no job backend is available.")
try:
active_job_backend.init()
except RuntimeError as err:
raise EasyBuildError("connection to server failed (%s: %s), can't submit jobs.", err.__class__.__name__, err)
# dependencies have already been resolved,
# so one can linearly walk over the list and use previous job id's
jobs = []
# keep track of which job builds which module
module_to_job = {}
for easyconfig in easyconfigs:
# this is very important, otherwise we might have race conditions
# e.g. GCC-4.5.3 finds cloog.tar.gz but it was incorrectly downloaded by GCC-4.6.3
# running this step here, prevents this
if prepare_first:
prepare_easyconfig(easyconfig)
# the new job will only depend on already submitted jobs
_log.info("creating job for ec: %s" % easyconfig['ec'])
new_job = create_job(active_job_backend, build_command, easyconfig, output_dir=output_dir)
# filter out dependencies marked as external modules
deps = [d for d in easyconfig['ec'].all_dependencies if not d.get('external_module', False)]
dep_mod_names = map(ActiveMNS().det_full_module_name, deps)
job_deps = [module_to_job[dep] for dep in dep_mod_names if dep in module_to_job]
# actually (try to) submit job
active_job_backend.queue(new_job, job_deps)
_log.info("job %s for module %s has been submitted", new_job, new_job.module)
# update dictionary
module_to_job[new_job.module] = new_job
jobs.append(new_job)
active_job_backend.complete()
return jobs
def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybuild-build', prepare_first=True):
"""
Build easyconfigs in parallel by submitting jobs to a batch-queuing system.
Return list of jobs submitted.
Argument `easyconfigs` is a list of easyconfigs which can be
built: e.g. they have no unresolved dependencies. This function
will build them in parallel by submitting jobs.
:param build_command: build command to use
:param easyconfigs: list of easyconfig files
:param output_dir: output directory
:param prepare_first: prepare by runnning fetch step first for each easyconfig
"""
_log.info("going to build these easyconfigs in parallel: %s", easyconfigs)
active_job_backend = job_backend()
if active_job_backend is None:
raise EasyBuildError("Can not use --job if no job backend is available.")
try:
active_job_backend.init()
except RuntimeError as err:
raise EasyBuildError("connection to server failed (%s: %s), can't submit jobs.", err.__class__.__name__, err)
# dependencies have already been resolved,
# so one can linearly walk over the list and use previous job id's
jobs = []
# keep track of which job builds which module
module_to_job = {}
for easyconfig in easyconfigs:
# this is very important, otherwise we might have race conditions
# e.g. GCC-4.5.3 finds cloog.tar.gz but it was incorrectly downloaded by GCC-4.6.3
# running this step here, prevents this
if prepare_first:
prepare_easyconfig(easyconfig)
# the new job will only depend on already submitted jobs
_log.info("creating job for ec: %s" % easyconfig['ec'])
new_job = create_job(active_job_backend, build_command, easyconfig, output_dir=output_dir)
# filter out dependencies marked as external modules
deps = [d for d in easyconfig['ec'].all_dependencies if not d.get('external_module', False)]
dep_mod_names = map(ActiveMNS().det_full_module_name, deps)
job_deps = [module_to_job[dep] for dep in dep_mod_names if dep in module_to_job]
# actually (try to) submit job
active_job_backend.queue(new_job, job_deps)
_log.info("job %s for module %s has been submitted", new_job, new_job.module)
# update dictionary
module_to_job[new_job.module] = new_job
jobs.append(new_job)
active_job_backend.complete()
return jobs