def run_parallel(fn_name, items, metadata=None):
items = [x for x in items if x is not None]
items = diagnostics.track_parallel(items, fn_name)
imodule = parallel.get("module", "bcbio.distributed")
if parallel["type"].startswith("messaging"):
task_module = "{base}.tasks".format(base=imodule)
runner_fn = runner(task_module, dirs, config, config_file)
return runner_fn(fn_name, items)
elif parallel["type"] == "ipython":
return ipython.runner(parallel, fn_name, items, dirs["work"],
config)
else:
logger.info("multiprocessing: %s" % fn_name)
fn = getattr(
__import__("{base}.multitasks".format(base=imodule),
fromlist=["multitasks"]), fn_name)
num_jobs, cores_per_job = ipython.find_cores_per_job([fn],
parallel,
items, config)
items = [
ipython.add_cores_to_config(x, cores_per_job) for x in items
]
num_jobs = cores_including_resources(num_jobs, metadata, config)
if joblib is None:
raise ImportError(
"Need joblib for multiprocessing parallelization")
out = []
for data in joblib.Parallel(num_jobs)(joblib.delayed(fn)(x)
for x in items):
if data:
out.extend(data)
return out
def run_parallel(fn_name, items, metadata=None):
items = [x for x in items if x is not None]
if len(items) == 0:
return []
items = diagnostics.track_parallel(items, fn_name)
imodule = parallel.get("module", "bcbio.distributed")
sysinfo = system.get_info(dirs, parallel)
if parallel["type"].startswith("messaging"):
task_module = "{base}.tasks".format(base=imodule)
runner_fn = runner(task_module, dirs, config, config_file)
return runner_fn(fn_name, items)
elif parallel["type"] == "ipython":
return ipython.runner(parallel, fn_name, items, dirs["work"], sysinfo, config)
else:
logger.info("multiprocessing: %s" % fn_name)
fn = getattr(__import__("{base}.multitasks".format(base=imodule),
fromlist=["multitasks"]),
fn_name)
jobr = ipython.find_job_resources([fn], parallel, items, sysinfo, config)
items = [ipython.add_cores_to_config(x, jobr.cores_per_job) for x in items]
if joblib is None:
raise ImportError("Need joblib for multiprocessing parallelization")
out = []
for data in joblib.Parallel(jobr.num_jobs)(joblib.delayed(fn)(x) for x in items):
if data:
out.extend(data)
return out
def run(fn_name, items):
setpath.prepend_bcbiopath()
out = []
fn, fn_name = (fn_name, fn_name.__name__) if callable(fn_name) else (
_get_ipython_fn(fn_name, parallel), fn_name)
items = [x for x in items if x is not None]
items = diagnostics.track_parallel(items, fn_name)
logger.info("ipython: %s" % fn_name)
if len(items) > 0:
items = [
config_utils.add_cores_to_config(x, parallel["cores_per_job"],
parallel) for x in items
]
if "wrapper" in parallel:
wrap_parallel = {
k: v
for k, v in parallel.items() if k in set(["fresources"])
}
items = [[fn_name] + parallel.get("wrapper_args", []) +
[wrap_parallel] + list(x) for x in items]
items = zip_args([args for args in items])
for data in view.map_sync(fn, items, track=False):
if data:
out.extend(unzip_args(data))
return out
def run_parallel(fn_name, items):
items = [x for x in items if x is not None]
if len(items) == 0:
return []
items = diagnostics.track_parallel(items, fn_name)
logger.info("multiprocessing: %s" % fn_name)
fn = get_fn(fn_name, parallel)
if "wrapper" in parallel:
items = [[fn_name] + parallel.get("wrapper_args", []) + list(x) for x in items]
return run_multicore(fn, items, config, parallel=parallel)
def run_parallel(fn_name, items):
items = [x for x in items if x is not None]
if len(items) == 0:
return []
items = diagnostics.track_parallel(items, fn_name)
fn, fn_name = (fn_name, fn_name.__name__) if callable(fn_name) else (get_fn(fn_name, parallel), fn_name)
logger.info("multiprocessing: %s" % fn_name)
if "wrapper" in parallel:
wrap_parallel = {k: v for k, v in parallel.items() if k in set(["fresources", "checkpointed"])}
items = [[fn_name] + parallel.get("wrapper_args", []) + [wrap_parallel] + list(x) for x in items]
return run_multicore(fn, items, config, parallel=parallel)
def run_parallel(fn_name, items):
items = [x for x in items if x is not None]
if len(items) == 0:
return []
items = diagnostics.track_parallel(items, fn_name)
fn, fn_name = (fn_name, fn_name.__name__) if callable(fn_name) else (get_fn(fn_name, parallel), fn_name)
logger.info("multiprocessing: %s" % fn_name)
if "wrapper" in parallel:
wrap_parallel = {k: v for k, v in parallel.items() if k in set(["fresources", "checkpointed"])}
items = [[fn_name] + parallel.get("wrapper_args", []) + [wrap_parallel] + list(x) for x in items]
return run_multicore(fn, items, config, parallel=parallel)
def run(fn_name, items):
out = []
items = [x for x in items if x is not None]
items = diagnostics.track_parallel(items, fn_name)
fn = _get_ipython_fn(fn_name, parallel)
logger.info("ipython: %s" % fn_name)
if len(items) > 0:
items = [config_utils.add_cores_to_config(x, parallel["cores_per_job"], parallel) for x in items]
if "wrapper" in parallel:
items = [[fn_name] + parallel.get("wrapper_args", []) + list(x) for x in items]
for data in view.map_sync(fn, items, track=False):
if data:
out.extend(data)
return out
def run_parallel(fn_name, items, metadata=None):
items = [x for x in items if x is not None]
if len(items) == 0:
return []
items = diagnostics.track_parallel(items, fn_name)
sysinfo = system.get_info(dirs, parallel)
if parallel["type"] == "ipython":
return ipython.runner(parallel, fn_name, items, dirs["work"], sysinfo, config)
else:
imodule = parallel.get("module", "bcbio.distributed")
logger.info("multiprocessing: %s" % fn_name)
fn = getattr(__import__("{base}.multitasks".format(base=imodule),
fromlist=["multitasks"]),
fn_name)
return run_multicore(fn, items, config, parallel["cores"])
def run(fn_name, items):
out = []
items = [x for x in items if x is not None]
items = diagnostics.track_parallel(items, fn_name)
fn = _get_ipython_fn(fn_name, parallel)
logger.info("ipython: %s" % fn_name)
if len(items) > 0:
items = [config_utils.add_cores_to_config(x, parallel["cores_per_job"], parallel) for x in items]
if "wrapper" in parallel:
wrap_parallel = {k: v for k, v in parallel.items() if k in set(["fresources"])}
items = [[fn_name] + parallel.get("wrapper_args", []) + [wrap_parallel] + list(x) for x in items]
items = zip_args([args for args in items])
for data in view.map_sync(fn, items, track=False):
if data:
out.extend(unzip_args(data))
return out
def run_parallel(fn_name, items, metadata=None):
items = [x for x in items if x is not None]
if len(items) == 0:
return []
items = diagnostics.track_parallel(items, fn_name)
imodule = parallel.get("module", "bcbio.distributed")
sysinfo = system.get_info(dirs, parallel)
if parallel["type"].startswith("messaging"):
raise NotImplementedError("Messaging parallelization no longer supported")
task_module = "{base}.tasks".format(base=imodule)
runner_fn = runner(task_module, dirs, config, config_file)
return runner_fn(fn_name, items)
elif parallel["type"] == "ipython":
return ipython.runner(parallel, fn_name, items, dirs["work"], sysinfo, config)
else:
logger.info("multiprocessing: %s" % fn_name)
fn = getattr(__import__("{base}.multitasks".format(base=imodule),
fromlist=["multitasks"]),
fn_name)
return run_multicore(fn, items, config, parallel["cores"])
def run(fn_name, items):
out = []
items = [x for x in items if x is not None]
items = diagnostics.track_parallel(items, fn_name)
fn = _get_ipython_fn(fn_name, parallel)
logger.info("ipython: %s" % fn_name)
if len(items) > 0:
items = [
config_utils.add_cores_to_config(x, parallel["cores_per_job"],
parallel) for x in items
]
if "wrapper" in parallel:
wrap_parallel = {
k: v
for k, v in parallel.items() if k in set(["fresources"])
}
items = [[fn_name] + parallel.get("wrapper_args", []) +
[wrap_parallel] + list(x) for x in items]
for data in view.map_sync(fn, items, track=False):
if data:
out.extend(data)
return out
def run_parallel(fn_name, items, metadata=None):
items = [x for x in items if x is not None]
items = diagnostics.track_parallel(items, fn_name)
if parallel["type"].startswith("messaging"):
task_module = "{base}.tasks".format(base=parallel["module"])
runner_fn = runner(task_module, dirs, config, config_file)
return runner_fn(fn_name, items)
elif parallel["type"] == "ipython":
return ipython.runner(parallel, fn_name, items, dirs["work"], config)
else:
logger.info("multiprocessing: %s" % fn_name)
out = []
fn = getattr(__import__("{base}.multitasks".format(base=parallel["module"]),
fromlist=["multitasks"]),
fn_name)
num_jobs, cores_per_job = ipython.find_cores_per_job(fn, parallel, items, config)
items = [ipython.add_cores_to_config(x, cores_per_job) for x in items]
num_jobs = cores_including_resources(num_jobs, metadata, config)
with utils.cpmap(num_jobs) as cpmap:
for data in cpmap(fn, items):
if data:
out.extend(data)
return out
