def _run_biodata_upload(args): """Manage preparation of biodata on a local machine, uploading to S3 in pieces. """ args = defaults.update_check_args(args, "biodata not uploaded") args = install.docker_image_arg(args) for gbuild in args.genomes: print("Preparing %s" % gbuild) if args.prepped: for target in ["samtools"] + args.aligners: genome.download_prepped_genome(gbuild, {}, target, False, args.prepped) print( "Downloaded prepped %s to %s. Edit and re-run without --prepped to upload" % (gbuild, args.prepped)) return cl = ["upgrade", "--genomes", gbuild] for a in args.aligners: cl += ["--aligners", a] for t in args.datatarget: cl += ["--datatarget", t] dmounts = mounts.prepare_system(args.datadir, DOCKER["biodata_dir"]) manage.run_bcbio_cmd(args.image, dmounts, cl) print("Uploading %s" % gbuild) gdir = _get_basedir(args.datadir, gbuild) basedir, genomedir = os.path.split(gdir) assert genomedir == gbuild with utils.chdir(basedir): all_dirs = sorted(os.listdir(gbuild)) _upload_biodata(gbuild, "seq", all_dirs) for aligner in args.aligners + ["rtg"]: _upload_biodata(gbuild, genome.REMAP_NAMES.get(aligner, aligner), all_dirs)
def cmd_server(args): args = defaults.update_check_args(args, "Could not run server.") args = install.docker_image_arg(args) ports = ["%s:%s" % (args.port, devel.DOCKER["port"])] print("Running server on port %s. Press ctrl-c to exit." % args.port) manage.run_bcbio_cmd(args.image, [], ["server", "--port", str(devel.DOCKER["port"])], ports)
def _run_biodata_upload(args): """Manage preparation of biodata on a local machine, uploading to S3 in pieces. """ args = defaults.update_check_args(args, "biodata not uploaded") args = install.docker_image_arg(args) for gbuild in args.genomes: print("Preparing %s" % gbuild) if args.prepped: for target in ["samtools"] + args.aligners: genome.download_prepped_genome(gbuild, {}, target, False, args.prepped) print("Downloaded prepped %s to %s. Edit and re-run without --prepped to upload" % (gbuild, args.prepped)) return cl = ["upgrade", "--genomes", gbuild] for a in args.aligners: cl += ["--aligners", a] dmounts = mounts.prepare_system(args.datadir, DOCKER["biodata_dir"]) manage.run_bcbio_cmd(args.image, dmounts, cl) print("Uploading %s" % gbuild) gdir = _get_basedir(args.datadir, gbuild) basedir, genomedir = os.path.split(gdir) assert genomedir == gbuild with utils.chdir(basedir): all_dirs = sorted(os.listdir(gbuild)) _upload_biodata(gbuild, "seq", all_dirs) for aligner in args.aligners: _upload_biodata(gbuild, genome.REMAP_NAMES.get(aligner, aligner), all_dirs)
def _run_system_update(args): """Update bcbio_system.yaml file with a given target of cores and memory. """ mem_types = set(["memory", "jvm_opts"]) args = defaults.update_check_args( args, "Could not do upgrade of bcbio_system.yaml") system_file = os.path.join(args.datadir, "galaxy", "bcbio_system.yaml") with open(system_file) as in_handle: config = yaml.safe_load(in_handle) out = copy.deepcopy(config) mems = [] for attrs in config.get("resources", {}).itervalues(): for key, value in attrs.iteritems(): if key in mem_types: mems.append((key, value)) common_mem = _calculate_common_memory(mems) for prog, attrs in config.get("resources", {}).iteritems(): for key, value in attrs.iteritems(): if key == "cores": out['resources'][prog][key] = int(args.cores) elif key in mem_types: out["resources"][prog][key] = _update_memory( key, value, args.memory, common_mem) bak_file = system_file + ".bak%s" % datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S") shutil.move(system_file, bak_file) with open(system_file, "w") as out_handle: yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False)
def cmd_ipython(args): args = defaults.update_check_args(args, "Could not run IPython parallel analysis.") args = install.docker_image_arg(args) parallel = clargs.to_parallel(args, "bcbiovm.docker") parallel["wrapper"] = "runfn" with open(args.sample_config) as in_handle: ready_config, _ = mounts.normalize_config(yaml.load(in_handle), args.fcdir) work_dir = os.getcwd() ready_config_file = os.path.join(work_dir, "%s-ready%s" % (os.path.splitext(os.path.basename(args.sample_config)))) with open(ready_config_file, "w") as out_handle: yaml.safe_dump(ready_config, out_handle, default_flow_style=False, allow_unicode=False) work_dir = os.getcwd() systemconfig = run.local_system_config(args.systemconfig, args.datadir, work_dir) cur_pack = pack.shared_filesystem(work_dir, args.datadir, args.tmpdir) parallel["wrapper_args"] = [devel.DOCKER, {"sample_config": ready_config_file, "fcdir": args.fcdir, "pack": cur_pack, "systemconfig": systemconfig, "image": args.image}] # For testing, run on a local ipython cluster parallel["run_local"] = parallel.get("queue") == "localrun" from bcbio.pipeline import main main.run_main(work_dir, run_info_yaml=ready_config_file, config_file=systemconfig, fc_dir=args.fcdir, parallel=parallel)
def cmd_ipython(args): args = defaults.update_check_args(args, "Could not run IPython parallel analysis.") args = install.docker_image_arg(args) parallel = clargs.to_parallel(args, "bcbiovm.docker") parallel["wrapper"] = "runfn" with open(args.sample_config) as in_handle: ready_config, _ = mounts.normalize_config(yaml.load(in_handle), args.fcdir) work_dir = os.getcwd() ready_config_file = os.path.join(work_dir, "%s-ready%s" % (os.path.splitext(os.path.basename(args.sample_config)))) with open(ready_config_file, "w") as out_handle: yaml.safe_dump(ready_config, out_handle, default_flow_style=False, allow_unicode=False) parallel["wrapper_args"] = [DOCKER, {"sample_config": ready_config_file, "fcdir": args.fcdir, "pack": pack.shared_filesystem(work_dir, args.datadir, args.tmpdir), "systemconfig": args.systemconfig, "image": args.image}] # For testing, run on a local ipython cluster parallel["run_local"] = parallel.get("queue") == "localrun" workdir_mount = "%s:%s" % (work_dir, DOCKER["work_dir"]) manage.run_bcbio_cmd(args.image, [workdir_mount], ["version", "--workdir=%s" % DOCKER["work_dir"]]) main.run_main(work_dir, run_info_yaml=ready_config_file, config_file=args.systemconfig, fc_dir=args.fcdir, parallel=parallel)
def cmd_ipython(args): args = defaults.update_check_args(args, "Could not run IPython parallel analysis.") args = install.docker_image_arg(args) parallel = clargs.to_parallel(args, "bcbiovm.docker") parallel["wrapper"] = "runfn" with open(args.sample_config) as in_handle: ready_config, _ = mounts.normalize_config(yaml.safe_load(in_handle), args.fcdir) work_dir = os.getcwd() ready_config_file = os.path.join(work_dir, "%s-ready%s" % (os.path.splitext(os.path.basename(args.sample_config)))) with open(ready_config_file, "w") as out_handle: yaml.safe_dump(ready_config, out_handle, default_flow_style=False, allow_unicode=False) work_dir = os.getcwd() systemconfig = run.local_system_config(args.systemconfig, args.datadir, work_dir) cur_pack = pack.shared_filesystem(work_dir, args.datadir, args.tmpdir) parallel["wrapper_args"] = [devel.DOCKER, {"sample_config": ready_config_file, "fcdir": args.fcdir, "pack": cur_pack, "systemconfig": systemconfig, "image": args.image}] # For testing, run on a local ipython cluster parallel["run_local"] = parallel.get("queue") == "localrun" from bcbio.pipeline import main main.run_main(work_dir, run_info_yaml=ready_config_file, config_file=systemconfig, fc_dir=args.fcdir, parallel=parallel)
def _run_system_update(args): """Update bcbio_system.yaml file with a given target of cores and memory. """ mem_types = set(["memory", "jvm_opts"]) args = defaults.update_check_args(args, "Could not do upgrade of bcbio_system.yaml") system_file = os.path.join(args.datadir, "galaxy", "bcbio_system.yaml") with open(system_file) as in_handle: config = yaml.safe_load(in_handle) out = copy.deepcopy(config) mems = [] for attrs in config.get("resources", {}).itervalues(): for key, value in attrs.iteritems(): if key in mem_types: mems.append((key, value)) common_mem = _calculate_common_memory(mems) for prog, attrs in config.get("resources", {}).iteritems(): for key, value in attrs.iteritems(): if key == "cores": out['resources'][prog][key] = int(args.cores) elif key in mem_types: out["resources"][prog][key] = _update_memory(key, value, args.memory, common_mem) bak_file = system_file + ".bak%s" % datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") shutil.move(system_file, bak_file) with open(system_file, "w") as out_handle: yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False)
def submit_script(args): args = defaults.update_check_args(args, "Could not prep batch scripts") out_file = os.path.join(os.getcwd(), "bcbio_submit.sh") with open(out_file, "w") as out_handle: out_handle.write("#!/bin/bash\n") out_handle.write(_get_scheduler_cmds(args) + "\n") out_handle.write(_get_ipython_cmdline(args) + "\n") print("Submission script for %s written to %s" % (args.scheduler, out_file)) print("Start analysis with: %s %s" % (_get_submit_cmd(args.scheduler), out_file))
def cmd_runfn(args): args = defaults.update_check_args(args, "Could not run bcbio-nextgen function.") with open(args.parallel) as in_handle: parallel = yaml.safe_load(in_handle) with open(args.runargs) as in_handle: runargs = yaml.safe_load(in_handle) cmd_args = {"systemconfig": args.systemconfig, "pack": parallel["pack"]} out = run.do_runfn(args.fn_name, runargs, cmd_args, parallel, DOCKER) out_file = "%s-out" % os.path.splitext(args.runargs) with open(out_file, "w") as out_handle: yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False)
def cmd_runfn(args): args = defaults.update_check_args(args, "Could not run bcbio-nextgen function.") args = install.docker_image_arg(args) with open(args.parallel) as in_handle: parallel = yaml.safe_load(in_handle) with open(args.runargs) as in_handle: runargs = yaml.safe_load(in_handle) cmd_args = {"systemconfig": args.systemconfig, "image": args.image, "pack": parallel["pack"]} out = run.do_runfn(args.fn_name, runargs, cmd_args, parallel, devel.DOCKER) out_file = "%s-out%s" % os.path.splitext(args.runargs) with open(out_file, "w") as out_handle: yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False) pack.send_output(parallel["pack"], out_file)
def _run_biodata_upload(args): """Manage preparation of biodata on a local machine, uploading to S3 in pieces. """ args = defaults.update_check_args(args, "biodata not uploaded") args = install.docker_image_arg(args) for gbuild in args.genomes: print("Preparing %s" % gbuild) cl = ["upgrade", "--genomes", gbuild] for a in args.aligners: cl += ["--aligners", a] dmounts = mounts.prepare_system(args.datadir, DOCKER["biodata_dir"]) manage.run_bcbio_cmd(args.image, dmounts, cl) print("Uploading %s" % gbuild) gdir = _get_basedir(args.datadir, gbuild) basedir, genomedir = os.path.split(gdir) assert genomedir == gbuild with utils.chdir(basedir): all_dirs = sorted(os.listdir(gbuild)) _upload_biodata(gbuild, "seq", all_dirs) for aligner in args.aligners: _upload_biodata(gbuild, genome.REMAP_NAMES.get(aligner, aligner), all_dirs)
def cmd_run(args): args = defaults.update_check_args(args, "Could not run analysis.") args = install.docker_image_arg(args) run.do_analysis(args, devel.DOCKER)
def cmd_clusterk(args): args = defaults.update_check_args( args, "Could not run Clusterk parallel analysis.") args = install.docker_image_arg(args) clusterk_main.run(args, devel.DOCKER)
def cmd_run(args): args = defaults.update_check_args(args, "Could not run analysis.") run.do_analysis(args, DOCKER)
def cmd_install(args): args = defaults.update_check_args(args, "bcbio-nextgen not upgraded.", need_datadir=args.install_data) install.full(args, devel.DOCKER)
def cmd_clusterk(args): args = defaults.update_check_args(args, "Could not run Clusterk parallel analysis.") clusterk_main.run(args, DOCKER)
def cmd_server(args): args = defaults.update_check_args(args, "Could not run server.") ports = ["%s:%s" % (args.port, DOCKER["port"])] print("Running server on port %s. Press ctrl-c to exit." % args.port) manage.run_bcbio_cmd(DOCKER["image"], [], ["server", "--port", str(DOCKER["port"])], ports)
def cmd_clusterk(args): args = defaults.update_check_args(args, "Could not run Clusterk parallel analysis.") args = install.docker_image_arg(args) clusterk_main.run(args, devel.DOCKER)