def run_parallel(fn_name, items, metadata=None): items = [x for x in items if x is not None] items = diagnostics.track_parallel(items, fn_name) imodule = parallel.get("module", "bcbio.distributed") if parallel["type"].startswith("messaging"): task_module = "{base}.tasks".format(base=imodule) runner_fn = runner(task_module, dirs, config, config_file) return runner_fn(fn_name, items) elif parallel["type"] == "ipython": return ipython.runner(parallel, fn_name, items, dirs["work"], config) else: logger.info("multiprocessing: %s" % fn_name) fn = getattr( __import__("{base}.multitasks".format(base=imodule), fromlist=["multitasks"]), fn_name) num_jobs, cores_per_job = ipython.find_cores_per_job([fn], parallel, items, config) items = [ ipython.add_cores_to_config(x, cores_per_job) for x in items ] num_jobs = cores_including_resources(num_jobs, metadata, config) if joblib is None: raise ImportError( "Need joblib for multiprocessing parallelization") out = [] for data in joblib.Parallel(num_jobs)(joblib.delayed(fn)(x) for x in items): if data: out.extend(data) return out
def run_parallel(fn_name, items, metadata=None): items = [x for x in items if x is not None] if len(items) == 0: return [] items = diagnostics.track_parallel(items, fn_name) imodule = parallel.get("module", "bcbio.distributed") sysinfo = system.get_info(dirs, parallel) if parallel["type"].startswith("messaging"): task_module = "{base}.tasks".format(base=imodule) runner_fn = runner(task_module, dirs, config, config_file) return runner_fn(fn_name, items) elif parallel["type"] == "ipython": return ipython.runner(parallel, fn_name, items, dirs["work"], sysinfo, config) else: logger.info("multiprocessing: %s" % fn_name) fn = getattr(__import__("{base}.multitasks".format(base=imodule), fromlist=["multitasks"]), fn_name) jobr = ipython.find_job_resources([fn], parallel, items, sysinfo, config) items = [ipython.add_cores_to_config(x, jobr.cores_per_job) for x in items] if joblib is None: raise ImportError("Need joblib for multiprocessing parallelization") out = [] for data in joblib.Parallel(jobr.num_jobs)(joblib.delayed(fn)(x) for x in items): if data: out.extend(data) return out
def run(fn_name, items): setpath.prepend_bcbiopath() out = [] fn, fn_name = (fn_name, fn_name.__name__) if callable(fn_name) else ( _get_ipython_fn(fn_name, parallel), fn_name) items = [x for x in items if x is not None] items = diagnostics.track_parallel(items, fn_name) logger.info("ipython: %s" % fn_name) if len(items) > 0: items = [ config_utils.add_cores_to_config(x, parallel["cores_per_job"], parallel) for x in items ] if "wrapper" in parallel: wrap_parallel = { k: v for k, v in parallel.items() if k in set(["fresources"]) } items = [[fn_name] + parallel.get("wrapper_args", []) + [wrap_parallel] + list(x) for x in items] items = zip_args([args for args in items]) for data in view.map_sync(fn, items, track=False): if data: out.extend(unzip_args(data)) return out
def run_parallel(fn_name, items): items = [x for x in items if x is not None] if len(items) == 0: return [] items = diagnostics.track_parallel(items, fn_name) logger.info("multiprocessing: %s" % fn_name) fn = get_fn(fn_name, parallel) if "wrapper" in parallel: items = [[fn_name] + parallel.get("wrapper_args", []) + list(x) for x in items] return run_multicore(fn, items, config, parallel=parallel)
def run_parallel(fn_name, items): items = [x for x in items if x is not None] if len(items) == 0: return [] items = diagnostics.track_parallel(items, fn_name) fn, fn_name = (fn_name, fn_name.__name__) if callable(fn_name) else (get_fn(fn_name, parallel), fn_name) logger.info("multiprocessing: %s" % fn_name) if "wrapper" in parallel: wrap_parallel = {k: v for k, v in parallel.items() if k in set(["fresources", "checkpointed"])} items = [[fn_name] + parallel.get("wrapper_args", []) + [wrap_parallel] + list(x) for x in items] return run_multicore(fn, items, config, parallel=parallel)
def run(fn_name, items): out = [] items = [x for x in items if x is not None] items = diagnostics.track_parallel(items, fn_name) fn = _get_ipython_fn(fn_name, parallel) logger.info("ipython: %s" % fn_name) if len(items) > 0: items = [config_utils.add_cores_to_config(x, parallel["cores_per_job"], parallel) for x in items] if "wrapper" in parallel: items = [[fn_name] + parallel.get("wrapper_args", []) + list(x) for x in items] for data in view.map_sync(fn, items, track=False): if data: out.extend(data) return out
def run_parallel(fn_name, items, metadata=None): items = [x for x in items if x is not None] if len(items) == 0: return [] items = diagnostics.track_parallel(items, fn_name) sysinfo = system.get_info(dirs, parallel) if parallel["type"] == "ipython": return ipython.runner(parallel, fn_name, items, dirs["work"], sysinfo, config) else: imodule = parallel.get("module", "bcbio.distributed") logger.info("multiprocessing: %s" % fn_name) fn = getattr(__import__("{base}.multitasks".format(base=imodule), fromlist=["multitasks"]), fn_name) return run_multicore(fn, items, config, parallel["cores"])
def run(fn_name, items): out = [] items = [x for x in items if x is not None] items = diagnostics.track_parallel(items, fn_name) fn = _get_ipython_fn(fn_name, parallel) logger.info("ipython: %s" % fn_name) if len(items) > 0: items = [config_utils.add_cores_to_config(x, parallel["cores_per_job"], parallel) for x in items] if "wrapper" in parallel: wrap_parallel = {k: v for k, v in parallel.items() if k in set(["fresources"])} items = [[fn_name] + parallel.get("wrapper_args", []) + [wrap_parallel] + list(x) for x in items] items = zip_args([args for args in items]) for data in view.map_sync(fn, items, track=False): if data: out.extend(unzip_args(data)) return out
def run_parallel(fn_name, items, metadata=None): items = [x for x in items if x is not None] if len(items) == 0: return [] items = diagnostics.track_parallel(items, fn_name) imodule = parallel.get("module", "bcbio.distributed") sysinfo = system.get_info(dirs, parallel) if parallel["type"].startswith("messaging"): raise NotImplementedError("Messaging parallelization no longer supported") task_module = "{base}.tasks".format(base=imodule) runner_fn = runner(task_module, dirs, config, config_file) return runner_fn(fn_name, items) elif parallel["type"] == "ipython": return ipython.runner(parallel, fn_name, items, dirs["work"], sysinfo, config) else: logger.info("multiprocessing: %s" % fn_name) fn = getattr(__import__("{base}.multitasks".format(base=imodule), fromlist=["multitasks"]), fn_name) return run_multicore(fn, items, config, parallel["cores"])
def run(fn_name, items): out = [] items = [x for x in items if x is not None] items = diagnostics.track_parallel(items, fn_name) fn = _get_ipython_fn(fn_name, parallel) logger.info("ipython: %s" % fn_name) if len(items) > 0: items = [ config_utils.add_cores_to_config(x, parallel["cores_per_job"], parallel) for x in items ] if "wrapper" in parallel: wrap_parallel = { k: v for k, v in parallel.items() if k in set(["fresources"]) } items = [[fn_name] + parallel.get("wrapper_args", []) + [wrap_parallel] + list(x) for x in items] for data in view.map_sync(fn, items, track=False): if data: out.extend(data) return out
def run_parallel(fn_name, items, metadata=None): items = [x for x in items if x is not None] items = diagnostics.track_parallel(items, fn_name) if parallel["type"].startswith("messaging"): task_module = "{base}.tasks".format(base=parallel["module"]) runner_fn = runner(task_module, dirs, config, config_file) return runner_fn(fn_name, items) elif parallel["type"] == "ipython": return ipython.runner(parallel, fn_name, items, dirs["work"], config) else: logger.info("multiprocessing: %s" % fn_name) out = [] fn = getattr(__import__("{base}.multitasks".format(base=parallel["module"]), fromlist=["multitasks"]), fn_name) num_jobs, cores_per_job = ipython.find_cores_per_job(fn, parallel, items, config) items = [ipython.add_cores_to_config(x, cores_per_job) for x in items] num_jobs = cores_including_resources(num_jobs, metadata, config) with utils.cpmap(num_jobs) as cpmap: for data in cpmap(fn, items): if data: out.extend(data) return out