def build_easyconfigs_in_parallel(build_command,
easyconfigs,
output_dir,
robot_path=None):
"""
easyconfigs is a list of easyconfigs which can be built (e.g. they have no unresolved dependencies)
this function will build them in parallel by submitting jobs
returns the jobs
"""
_log.info("going to build these easyconfigs in parallel: %s", easyconfigs)
job_module_dict = {}
# dependencies have already been resolved,
# so one can linearly walk over the list and use previous job id's
jobs = []
# create a single connection, and reuse it
conn = connect_to_server()
if conn is None:
_log.error("connect_to_server returned %s, can't submit jobs." %
(conn))
# determine ppn once, and pass is to each job being created
# this avoids having to figure out ppn over and over again, every time creating a temp connection to the server
ppn = get_ppn()
for ec in easyconfigs:
# This is very important, otherwise we might have race conditions
# e.g. GCC-4.5.3 finds cloog.tar.gz but it was incorrectly downloaded by GCC-4.6.3
# running this step here, prevents this
prepare_easyconfig(ec, robot_path=robot_path)
# the new job will only depend on already submitted jobs
_log.info("creating job for ec: %s" % str(ec))
new_job = create_job(build_command, ec, output_dir, conn=conn, ppn=ppn)
# Sometimes unresolvedDependencies will contain things, not needed to be build.
job_deps = [
job_module_dict[dep] for dep in ec['unresolvedDependencies']
if dep in job_module_dict
]
new_job.add_dependencies(job_deps)
new_job.submit()
_log.info("job for module %s has been submitted (job id: %s)" %
(new_job.module, new_job.jobid))
# update dictionary
job_module_dict[new_job.module] = new_job.jobid
new_job.cleanup()
jobs.append(new_job)
disconnect_from_server(conn)
return jobs
def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir, robot_path=None):
"""
easyconfigs is a list of easyconfigs which can be built (e.g. they have no unresolved dependencies)
this function will build them in parallel by submitting jobs
returns the jobs
"""
_log.info("going to build these easyconfigs in parallel: %s", easyconfigs)
job_ids = {}
# dependencies have already been resolved,
# so one can linearly walk over the list and use previous job id's
jobs = []
# create a single connection, and reuse it
conn = connect_to_server()
if conn is None:
_log.error("connect_to_server returned %s, can't submit jobs." % (conn))
# determine ppn once, and pass is to each job being created
# this avoids having to figure out ppn over and over again, every time creating a temp connection to the server
ppn = get_ppn()
def tokey(dep):
"""Determine key for specified dependency."""
return det_full_module_name(dep)
for ec in easyconfigs:
# This is very important, otherwise we might have race conditions
# e.g. GCC-4.5.3 finds cloog.tar.gz but it was incorrectly downloaded by GCC-4.6.3
# running this step here, prevents this
prepare_easyconfig(ec, robot_path=robot_path)
# the new job will only depend on already submitted jobs
_log.info("creating job for ec: %s" % str(ec))
new_job = create_job(build_command, ec, output_dir, conn=conn, ppn=ppn)
# sometimes unresolved_deps will contain things, not needed to be build
job_deps = [job_ids[dep] for dep in map(tokey, ec['unresolved_deps']) if dep in job_ids]
new_job.add_dependencies(job_deps)
new_job.submit()
_log.info("job for module %s has been submitted (job id: %s)" % (new_job.module, new_job.jobid))
# update dictionary
job_ids[new_job.module] = new_job.jobid
new_job.cleanup()
jobs.append(new_job)
disconnect_from_server(conn)
return jobs
def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir=None, prepare_first=True):
"""
easyconfigs is a list of easyconfigs which can be built (e.g. they have no unresolved dependencies)
this function will build them in parallel by submitting jobs
@param build_command: build command to use
@param easyconfigs: list of easyconfig files
@param output_dir: output directory
returns the jobs
"""
_log.info("going to build these easyconfigs in parallel: %s", easyconfigs)
job_ids = {}
# dependencies have already been resolved,
# so one can linearly walk over the list and use previous job id's
jobs = []
# create a single connection, and reuse it
conn = connect_to_server()
if conn is None:
_log.error("connect_to_server returned %s, can't submit jobs." % (conn))
# determine ppn once, and pass is to each job being created
# this avoids having to figure out ppn over and over again, every time creating a temp connection to the server
ppn = get_ppn()
def tokey(dep):
"""Determine key for specified dependency."""
return ActiveMNS().det_full_module_name(dep)
for ec in easyconfigs:
# this is very important, otherwise we might have race conditions
# e.g. GCC-4.5.3 finds cloog.tar.gz but it was incorrectly downloaded by GCC-4.6.3
# running this step here, prevents this
if prepare_first:
prepare_easyconfig(ec)
# the new job will only depend on already submitted jobs
_log.info("creating job for ec: %s" % str(ec))
new_job = create_job(build_command, ec, output_dir=output_dir, conn=conn, ppn=ppn)
# sometimes unresolved_deps will contain things, not needed to be build
job_deps = [job_ids[dep] for dep in map(tokey, ec['unresolved_deps']) if dep in job_ids]
new_job.add_dependencies(job_deps)
# place user hold on job to prevent it from starting too quickly,
# we might still need it in the queue to set it as a dependency for another job;
# only set hold for job without dependencies, other jobs have a dependency hold set anyway
with_hold = False
if not job_deps:
with_hold = True
# actually (try to) submit job
new_job.submit(with_hold)
_log.info("job for module %s has been submitted (job id: %s)" % (new_job.module, new_job.jobid))
# update dictionary
job_ids[new_job.module] = new_job.jobid
new_job.cleanup()
jobs.append(new_job)
# release all user holds on jobs after submission is completed
for job in jobs:
if job.has_holds():
_log.info("releasing hold on job %s" % job.jobid)
job.release_hold()
disconnect_from_server(conn)
return jobs
def build_easyconfigs_in_parallel(build_command,
easyconfigs,
output_dir=None,
prepare_first=True):
"""
easyconfigs is a list of easyconfigs which can be built (e.g. they have no unresolved dependencies)
this function will build them in parallel by submitting jobs
@param build_command: build command to use
@param easyconfigs: list of easyconfig files
@param output_dir: output directory
returns the jobs
"""
_log.info("going to build these easyconfigs in parallel: %s", easyconfigs)
job_ids = {}
# dependencies have already been resolved,
# so one can linearly walk over the list and use previous job id's
jobs = []
# create a single connection, and reuse it
conn = connect_to_server()
if conn is None:
raise EasyBuildError(
"connect_to_server returned %s, can't submit jobs.", conn)
# determine ppn once, and pass is to each job being created
# this avoids having to figure out ppn over and over again, every time creating a temp connection to the server
ppn = get_ppn()
def tokey(dep):
"""Determine key for specified dependency."""
return ActiveMNS().det_full_module_name(dep)
for ec in easyconfigs:
# this is very important, otherwise we might have race conditions
# e.g. GCC-4.5.3 finds cloog.tar.gz but it was incorrectly downloaded by GCC-4.6.3
# running this step here, prevents this
if prepare_first:
prepare_easyconfig(ec)
# the new job will only depend on already submitted jobs
_log.info("creating job for ec: %s" % str(ec))
new_job = create_job(build_command,
ec,
output_dir=output_dir,
conn=conn,
ppn=ppn)
# sometimes unresolved_deps will contain things, not needed to be build
job_deps = [
job_ids[dep] for dep in map(tokey, ec['unresolved_deps'])
if dep in job_ids
]
new_job.add_dependencies(job_deps)
# place user hold on job to prevent it from starting too quickly,
# we might still need it in the queue to set it as a dependency for another job;
# only set hold for job without dependencies, other jobs have a dependency hold set anyway
with_hold = False
if not job_deps:
with_hold = True
# actually (try to) submit job
new_job.submit(with_hold)
_log.info("job for module %s has been submitted (job id: %s)" %
(new_job.module, new_job.jobid))
# update dictionary
job_ids[new_job.module] = new_job.jobid
new_job.cleanup()
jobs.append(new_job)
# release all user holds on jobs after submission is completed
for job in jobs:
if job.has_holds():
_log.info("releasing hold on job %s" % job.jobid)
job.release_hold()
disconnect_from_server(conn)
return jobs
