def run_parallel(fn_name, items, metadata=None): items = [x for x in items if x is not None] items = diagnostics.track_parallel(items, fn_name) imodule = parallel.get("module", "bcbio.distributed") if parallel["type"].startswith("messaging"): task_module = "{base}.tasks".format(base=imodule) runner_fn = runner(task_module, dirs, config, config_file) return runner_fn(fn_name, items) elif parallel["type"] == "ipython": return ipython.runner(parallel, fn_name, items, dirs["work"], config) else: logger.info("multiprocessing: %s" % fn_name) fn = getattr( __import__("{base}.multitasks".format(base=imodule), fromlist=["multitasks"]), fn_name) num_jobs, cores_per_job = ipython.find_cores_per_job([fn], parallel, items, config) items = [ ipython.add_cores_to_config(x, cores_per_job) for x in items ] num_jobs = cores_including_resources(num_jobs, metadata, config) if joblib is None: raise ImportError( "Need joblib for multiprocessing parallelization") out = [] for data in joblib.Parallel(num_jobs)(joblib.delayed(fn)(x) for x in items): if data: out.extend(data) return out
def run_parallel(fn_name, items, metadata=None): items = [x for x in items if x is not None] items = diagnostics.track_parallel(items, fn_name) imodule = parallel.get("module", "bcbio.distributed") if parallel["type"].startswith("messaging"): task_module = "{base}.tasks".format(base=imodule) runner_fn = runner(task_module, dirs, config, config_file) return runner_fn(fn_name, items) elif parallel["type"] == "ipython": return ipython.runner(parallel, fn_name, items, dirs["work"], config) else: logger.info("multiprocessing: %s" % fn_name) fn = getattr(__import__("{base}.multitasks".format(base=imodule), fromlist=["multitasks"]), fn_name) num_jobs, cores_per_job = ipython.find_cores_per_job([fn], parallel, items, config) items = [ipython.add_cores_to_config(x, cores_per_job) for x in items] num_jobs = cores_including_resources(num_jobs, metadata, config) if joblib is None: raise ImportError("Need joblib for multiprocessing parallelization") out = [] for data in joblib.Parallel(num_jobs)(joblib.delayed(fn)(x) for x in items): if data: out.extend(data) return out
def run_parallel(fn_name, items, metadata=None): items = [x for x in items if x is not None] items = diagnostics.track_parallel(items, fn_name) if parallel["type"].startswith("messaging"): task_module = "{base}.tasks".format(base=parallel["module"]) runner_fn = runner(task_module, dirs, config, config_file) return runner_fn(fn_name, items) elif parallel["type"] == "ipython": return ipython.runner(parallel, fn_name, items, dirs["work"], config) else: logger.info("multiprocessing: %s" % fn_name) out = [] fn = getattr(__import__("{base}.multitasks".format(base=parallel["module"]), fromlist=["multitasks"]), fn_name) num_jobs, cores_per_job = ipython.find_cores_per_job(fn, parallel, items, config) items = [ipython.add_cores_to_config(x, cores_per_job) for x in items] num_jobs = cores_including_resources(num_jobs, metadata, config) with utils.cpmap(num_jobs) as cpmap: for data in cpmap(fn, items): if data: out.extend(data) return out