def _run_biodata_upload(args):
"""Manage preparation of biodata on a local machine, uploading to S3 in pieces.
"""
args = defaults.update_check_args(args, "biodata not uploaded")
args = install.docker_image_arg(args)
for gbuild in args.genomes:
print("Preparing %s" % gbuild)
if args.prepped:
for target in ["samtools"] + args.aligners:
genome.download_prepped_genome(gbuild, {}, target, False,
args.prepped)
print(
"Downloaded prepped %s to %s. Edit and re-run without --prepped to upload"
% (gbuild, args.prepped))
return
cl = ["upgrade", "--genomes", gbuild]
for a in args.aligners:
cl += ["--aligners", a]
for t in args.datatarget:
cl += ["--datatarget", t]
dmounts = mounts.prepare_system(args.datadir, DOCKER["biodata_dir"])
manage.run_bcbio_cmd(args.image, dmounts, cl)
print("Uploading %s" % gbuild)
gdir = _get_basedir(args.datadir, gbuild)
basedir, genomedir = os.path.split(gdir)
assert genomedir == gbuild
with utils.chdir(basedir):
all_dirs = sorted(os.listdir(gbuild))
_upload_biodata(gbuild, "seq", all_dirs)
for aligner in args.aligners + ["rtg"]:
_upload_biodata(gbuild,
genome.REMAP_NAMES.get(aligner,
aligner), all_dirs)
def cmd_server(args):
args = defaults.update_check_args(args, "Could not run server.")
args = install.docker_image_arg(args)
ports = ["%s:%s" % (args.port, devel.DOCKER["port"])]
print("Running server on port %s. Press ctrl-c to exit." % args.port)
manage.run_bcbio_cmd(args.image, [], ["server", "--port", str(devel.DOCKER["port"])],
ports)
def _run_biodata_upload(args):
"""Manage preparation of biodata on a local machine, uploading to S3 in pieces.
"""
args = defaults.update_check_args(args, "biodata not uploaded")
args = install.docker_image_arg(args)
for gbuild in args.genomes:
print("Preparing %s" % gbuild)
if args.prepped:
for target in ["samtools"] + args.aligners:
genome.download_prepped_genome(gbuild, {}, target, False, args.prepped)
print("Downloaded prepped %s to %s. Edit and re-run without --prepped to upload"
% (gbuild, args.prepped))
return
cl = ["upgrade", "--genomes", gbuild]
for a in args.aligners:
cl += ["--aligners", a]
dmounts = mounts.prepare_system(args.datadir, DOCKER["biodata_dir"])
manage.run_bcbio_cmd(args.image, dmounts, cl)
print("Uploading %s" % gbuild)
gdir = _get_basedir(args.datadir, gbuild)
basedir, genomedir = os.path.split(gdir)
assert genomedir == gbuild
with utils.chdir(basedir):
all_dirs = sorted(os.listdir(gbuild))
_upload_biodata(gbuild, "seq", all_dirs)
for aligner in args.aligners:
_upload_biodata(gbuild, genome.REMAP_NAMES.get(aligner, aligner), all_dirs)
def cmd_server(args):
args = defaults.update_check_args(args, "Could not run server.")
args = install.docker_image_arg(args)
ports = ["%s:%s" % (args.port, devel.DOCKER["port"])]
print("Running server on port %s. Press ctrl-c to exit." % args.port)
manage.run_bcbio_cmd(args.image, [], ["server", "--port", str(devel.DOCKER["port"])],
ports)
def _run_system_update(args):
"""Update bcbio_system.yaml file with a given target of cores and memory.
"""
mem_types = set(["memory", "jvm_opts"])
args = defaults.update_check_args(
args, "Could not do upgrade of bcbio_system.yaml")
system_file = os.path.join(args.datadir, "galaxy", "bcbio_system.yaml")
with open(system_file) as in_handle:
config = yaml.safe_load(in_handle)
out = copy.deepcopy(config)
mems = []
for attrs in config.get("resources", {}).itervalues():
for key, value in attrs.iteritems():
if key in mem_types:
mems.append((key, value))
common_mem = _calculate_common_memory(mems)
for prog, attrs in config.get("resources", {}).iteritems():
for key, value in attrs.iteritems():
if key == "cores":
out['resources'][prog][key] = int(args.cores)
elif key in mem_types:
out["resources"][prog][key] = _update_memory(
key, value, args.memory, common_mem)
bak_file = system_file + ".bak%s" % datetime.datetime.now().strftime(
"%Y-%m-%d-%H-%M-%S")
shutil.move(system_file, bak_file)
with open(system_file, "w") as out_handle:
yaml.safe_dump(out,
out_handle,
default_flow_style=False,
allow_unicode=False)
def cmd_ipython(args):
args = defaults.update_check_args(args, "Could not run IPython parallel analysis.")
args = install.docker_image_arg(args)
parallel = clargs.to_parallel(args, "bcbiovm.docker")
parallel["wrapper"] = "runfn"
with open(args.sample_config) as in_handle:
ready_config, _ = mounts.normalize_config(yaml.load(in_handle), args.fcdir)
work_dir = os.getcwd()
ready_config_file = os.path.join(work_dir, "%s-ready%s" %
(os.path.splitext(os.path.basename(args.sample_config))))
with open(ready_config_file, "w") as out_handle:
yaml.safe_dump(ready_config, out_handle, default_flow_style=False, allow_unicode=False)
work_dir = os.getcwd()
systemconfig = run.local_system_config(args.systemconfig, args.datadir, work_dir)
cur_pack = pack.shared_filesystem(work_dir, args.datadir, args.tmpdir)
parallel["wrapper_args"] = [devel.DOCKER, {"sample_config": ready_config_file,
"fcdir": args.fcdir,
"pack": cur_pack,
"systemconfig": systemconfig,
"image": args.image}]
# For testing, run on a local ipython cluster
parallel["run_local"] = parallel.get("queue") == "localrun"
from bcbio.pipeline import main
main.run_main(work_dir, run_info_yaml=ready_config_file,
config_file=systemconfig, fc_dir=args.fcdir,
parallel=parallel)
def cmd_ipython(args):
args = defaults.update_check_args(args, "Could not run IPython parallel analysis.")
args = install.docker_image_arg(args)
parallel = clargs.to_parallel(args, "bcbiovm.docker")
parallel["wrapper"] = "runfn"
with open(args.sample_config) as in_handle:
ready_config, _ = mounts.normalize_config(yaml.load(in_handle), args.fcdir)
work_dir = os.getcwd()
ready_config_file = os.path.join(work_dir, "%s-ready%s" %
(os.path.splitext(os.path.basename(args.sample_config))))
with open(ready_config_file, "w") as out_handle:
yaml.safe_dump(ready_config, out_handle, default_flow_style=False, allow_unicode=False)
parallel["wrapper_args"] = [DOCKER, {"sample_config": ready_config_file,
"fcdir": args.fcdir,
"pack": pack.shared_filesystem(work_dir, args.datadir, args.tmpdir),
"systemconfig": args.systemconfig,
"image": args.image}]
# For testing, run on a local ipython cluster
parallel["run_local"] = parallel.get("queue") == "localrun"
workdir_mount = "%s:%s" % (work_dir, DOCKER["work_dir"])
manage.run_bcbio_cmd(args.image, [workdir_mount],
["version", "--workdir=%s" % DOCKER["work_dir"]])
main.run_main(work_dir, run_info_yaml=ready_config_file,
config_file=args.systemconfig, fc_dir=args.fcdir,
parallel=parallel)
def cmd_ipython(args):
args = defaults.update_check_args(args, "Could not run IPython parallel analysis.")
args = install.docker_image_arg(args)
parallel = clargs.to_parallel(args, "bcbiovm.docker")
parallel["wrapper"] = "runfn"
with open(args.sample_config) as in_handle:
ready_config, _ = mounts.normalize_config(yaml.safe_load(in_handle), args.fcdir)
work_dir = os.getcwd()
ready_config_file = os.path.join(work_dir, "%s-ready%s" %
(os.path.splitext(os.path.basename(args.sample_config))))
with open(ready_config_file, "w") as out_handle:
yaml.safe_dump(ready_config, out_handle, default_flow_style=False, allow_unicode=False)
work_dir = os.getcwd()
systemconfig = run.local_system_config(args.systemconfig, args.datadir, work_dir)
cur_pack = pack.shared_filesystem(work_dir, args.datadir, args.tmpdir)
parallel["wrapper_args"] = [devel.DOCKER, {"sample_config": ready_config_file,
"fcdir": args.fcdir,
"pack": cur_pack,
"systemconfig": systemconfig,
"image": args.image}]
# For testing, run on a local ipython cluster
parallel["run_local"] = parallel.get("queue") == "localrun"
from bcbio.pipeline import main
main.run_main(work_dir, run_info_yaml=ready_config_file,
config_file=systemconfig, fc_dir=args.fcdir,
parallel=parallel)
def _run_system_update(args):
"""Update bcbio_system.yaml file with a given target of cores and memory.
"""
mem_types = set(["memory", "jvm_opts"])
args = defaults.update_check_args(args, "Could not do upgrade of bcbio_system.yaml")
system_file = os.path.join(args.datadir, "galaxy", "bcbio_system.yaml")
with open(system_file) as in_handle:
config = yaml.safe_load(in_handle)
out = copy.deepcopy(config)
mems = []
for attrs in config.get("resources", {}).itervalues():
for key, value in attrs.iteritems():
if key in mem_types:
mems.append((key, value))
common_mem = _calculate_common_memory(mems)
for prog, attrs in config.get("resources", {}).iteritems():
for key, value in attrs.iteritems():
if key == "cores":
out['resources'][prog][key] = int(args.cores)
elif key in mem_types:
out["resources"][prog][key] = _update_memory(key, value, args.memory,
common_mem)
bak_file = system_file + ".bak%s" % datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
shutil.move(system_file, bak_file)
with open(system_file, "w") as out_handle:
yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False)
def submit_script(args):
args = defaults.update_check_args(args, "Could not prep batch scripts")
out_file = os.path.join(os.getcwd(), "bcbio_submit.sh")
with open(out_file, "w") as out_handle:
out_handle.write("#!/bin/bash\n")
out_handle.write(_get_scheduler_cmds(args) + "\n")
out_handle.write(_get_ipython_cmdline(args) + "\n")
print("Submission script for %s written to %s" % (args.scheduler, out_file))
print("Start analysis with: %s %s" % (_get_submit_cmd(args.scheduler), out_file))
def cmd_runfn(args):
args = defaults.update_check_args(args, "Could not run bcbio-nextgen function.")
with open(args.parallel) as in_handle:
parallel = yaml.safe_load(in_handle)
with open(args.runargs) as in_handle:
runargs = yaml.safe_load(in_handle)
cmd_args = {"systemconfig": args.systemconfig, "pack": parallel["pack"]}
out = run.do_runfn(args.fn_name, runargs, cmd_args, parallel, DOCKER)
out_file = "%s-out" % os.path.splitext(args.runargs)
with open(out_file, "w") as out_handle:
yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False)
def submit_script(args):
args = defaults.update_check_args(args, "Could not prep batch scripts")
out_file = os.path.join(os.getcwd(), "bcbio_submit.sh")
with open(out_file, "w") as out_handle:
out_handle.write("#!/bin/bash\n")
out_handle.write(_get_scheduler_cmds(args) + "\n")
out_handle.write(_get_ipython_cmdline(args) + "\n")
print("Submission script for %s written to %s" %
(args.scheduler, out_file))
print("Start analysis with: %s %s" %
(_get_submit_cmd(args.scheduler), out_file))
def cmd_runfn(args):
args = defaults.update_check_args(args, "Could not run bcbio-nextgen function.")
args = install.docker_image_arg(args)
with open(args.parallel) as in_handle:
parallel = yaml.safe_load(in_handle)
with open(args.runargs) as in_handle:
runargs = yaml.safe_load(in_handle)
cmd_args = {"systemconfig": args.systemconfig, "image": args.image, "pack": parallel["pack"]}
out = run.do_runfn(args.fn_name, runargs, cmd_args, parallel, devel.DOCKER)
out_file = "%s-out%s" % os.path.splitext(args.runargs)
with open(out_file, "w") as out_handle:
yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False)
pack.send_output(parallel["pack"], out_file)
def _run_biodata_upload(args):
"""Manage preparation of biodata on a local machine, uploading to S3 in pieces.
"""
args = defaults.update_check_args(args, "biodata not uploaded")
args = install.docker_image_arg(args)
for gbuild in args.genomes:
print("Preparing %s" % gbuild)
cl = ["upgrade", "--genomes", gbuild]
for a in args.aligners:
cl += ["--aligners", a]
dmounts = mounts.prepare_system(args.datadir, DOCKER["biodata_dir"])
manage.run_bcbio_cmd(args.image, dmounts, cl)
print("Uploading %s" % gbuild)
gdir = _get_basedir(args.datadir, gbuild)
basedir, genomedir = os.path.split(gdir)
assert genomedir == gbuild
with utils.chdir(basedir):
all_dirs = sorted(os.listdir(gbuild))
_upload_biodata(gbuild, "seq", all_dirs)
for aligner in args.aligners:
_upload_biodata(gbuild, genome.REMAP_NAMES.get(aligner, aligner), all_dirs)
def cmd_run(args):
args = defaults.update_check_args(args, "Could not run analysis.")
args = install.docker_image_arg(args)
run.do_analysis(args, devel.DOCKER)
def cmd_clusterk(args):
args = defaults.update_check_args(
args, "Could not run Clusterk parallel analysis.")
args = install.docker_image_arg(args)
clusterk_main.run(args, devel.DOCKER)
def cmd_run(args):
args = defaults.update_check_args(args, "Could not run analysis.")
run.do_analysis(args, DOCKER)
def cmd_install(args):
args = defaults.update_check_args(args,
"bcbio-nextgen not upgraded.",
need_datadir=args.install_data)
install.full(args, devel.DOCKER)
def cmd_clusterk(args):
args = defaults.update_check_args(args, "Could not run Clusterk parallel analysis.")
clusterk_main.run(args, DOCKER)
def cmd_server(args):
args = defaults.update_check_args(args, "Could not run server.")
ports = ["%s:%s" % (args.port, DOCKER["port"])]
print("Running server on port %s. Press ctrl-c to exit." % args.port)
manage.run_bcbio_cmd(DOCKER["image"], [], ["server", "--port", str(DOCKER["port"])],
ports)
def cmd_clusterk(args):
args = defaults.update_check_args(args, "Could not run Clusterk parallel analysis.")
args = install.docker_image_arg(args)
clusterk_main.run(args, devel.DOCKER)
def cmd_install(args):
args = defaults.update_check_args(args, "bcbio-nextgen not upgraded.",
need_datadir=args.install_data)
install.full(args, devel.DOCKER)
def cmd_run(args):
args = defaults.update_check_args(args, "Could not run analysis.")
args = install.docker_image_arg(args)
run.do_analysis(args, devel.DOCKER)
