def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybuild-build', prepare_first=True): """ Build easyconfigs in parallel by submitting jobs to a batch-queuing system. Return list of jobs submitted. Argument `easyconfigs` is a list of easyconfigs which can be built: e.g. they have no unresolved dependencies. This function will build them in parallel by submitting jobs. :param build_command: build command to use :param easyconfigs: list of easyconfig files :param output_dir: output directory :param prepare_first: prepare by runnning fetch step first for each easyconfig """ _log.info("going to build these easyconfigs in parallel: %s", easyconfigs) active_job_backend = job_backend() if active_job_backend is None: raise EasyBuildError("Can not use --job if no job backend is available.") try: active_job_backend.init() except RuntimeError as err: raise EasyBuildError("connection to server failed (%s: %s), can't submit jobs.", err.__class__.__name__, err) # dependencies have already been resolved, # so one can linearly walk over the list and use previous job id's jobs = [] # keep track of which job builds which module module_to_job = {} for easyconfig in easyconfigs: # this is very important, otherwise we might have race conditions # e.g. GCC-4.5.3 finds cloog.tar.gz but it was incorrectly downloaded by GCC-4.6.3 # running this step here, prevents this if prepare_first: prepare_easyconfig(easyconfig) # the new job will only depend on already submitted jobs _log.info("creating job for ec: %s" % easyconfig['ec']) new_job = create_job(active_job_backend, build_command, easyconfig, output_dir=output_dir) # filter out dependencies marked as external modules deps = [d for d in easyconfig['ec'].all_dependencies if not d.get('external_module', False)] dep_mod_names = map(ActiveMNS().det_full_module_name, deps) job_deps = [module_to_job[dep] for dep in dep_mod_names if dep in module_to_job] # actually (try to) submit job active_job_backend.queue(new_job, job_deps) _log.info("job %s for module %s has been submitted", new_job, new_job.module) # update dictionary module_to_job[new_job.module] = new_job jobs.append(new_job) active_job_backend.complete() return jobs