def launch_mapping(metadata):
threads = 4
args = get_mapping_args(metadata, threads)
print(f"Aligning {len(args)} fastq files...")
if len(args) == 0:
print("No fastq files found for mapping...")
return
njobs = min(len(args), 128)
# if njobs < len(args):
# # should use int(math.ceil(len(args)/128))) or something like that
# raise Exception("NEED TO ADJUST RUN TIME accordingly")
job = remote.run_remote(mapping.run_mapping,
_jobmanager(),
job_name="mapping",
args=args,
job_dir="output",
overwrite=True,
njobs=njobs,
queue="owners",
mem="48g",
cpus=threads)
print(jobmanagers.wait_for_jobs([job], progress=True, wait=5.0))
def launch_basecalling(metadata):
threads = 8
args = get_basecalling_args(metadata, threads)
if len(args) == 0:
print("No directories found for basecalling...")
return
print(f"Running basecalling on {len(args)} fast5 files...")
njobs = min(len(args), 128)
# if njobs < len(args):
# # should use int(math.ceil(len(args)/128))) or something like that
# raise Exception("NEED TO ADJUST RUN TIME accordingly")
job = remote.run_remote(basecalling.run_basecalling_locally,
_jobmanager(),
job_name="basecalling",
args=args,
job_dir="output",
overwrite=True,
njobs=njobs,
queue="msalit,owners",
cpus=threads,
mem=f"{8*threads}g")
print(jobmanagers.wait_for_jobs([job], progress=True, wait=5.0))
def map(self, fn, args):
from admiral import jobmanagers, remote
cluster_options = self.cluster_settings.cluster_options.copy()
scheduler = cluster_options.pop("scheduler")
jobmanager_class = jobmanagers.get_jobmanager(scheduler)
jobmanager = jobmanager_class(batch_dir=self.batch_dir,
log_dir=self.batch_dir)
if not "mem" in cluster_options:
cluster_options["mem"] = "16g"
if not "time" in cluster_options:
cluster_options["time"] = "12h"
jobs = []
#for i, arg in enumerate(args):
job_name = args[0].__class__.__name__
args = [[arg] for arg in args]
job = remote.run_remote(fn,
jobmanager,
job_name,
args=args,
array=True,
overwrite=True,
**cluster_options)
result = jobmanagers.wait_for_jobs([job], wait=5, progress=True)
if not result:
raise Exception("Some chunks failed to complete")
def launch_archiving(metadata):
args = get_archiving_args(metadata)
if len(args) == 0:
print("Archiving complete; skipping...")
return
njobs = min(len(args), 128)
job = remote.run_remote(
fast5_archives.archive_chunk, _jobmanager(),
job_name="archive_fast5s", args=args, job_dir="output",
overwrite=True, njobs=njobs, queue="owners", mem="8g")
print(jobmanagers.wait_for_jobs([job], progress=True, wait=5.0))
def launch_merge_bams(metadata):
args = get_merge_bams_args(metadata)
if len(args) == 0:
print("No bam files found to merge...")
return
njobs = min(len(args), 128)
# if njobs < len(args):
# # should use int(math.ceil(len(args)/128))) or something like that
# raise Exception("NEED TO ADJUST RUN TIME accordingly")
job = remote.run_remote(
mapping.merge_bams, _jobmanager(),
job_name="merge_bams", args=args, job_dir="output",
overwrite=True, njobs=njobs, queue="owners", cpus=4, mem="24g")
print(jobmanagers.wait_for_jobs([job], progress=True, wait=5.0))