def test_copy_files(tmpdir):
from toil_scripts.lib.files import copy_files
work_dir = str(tmpdir)
os.mkdir(os.path.join(work_dir, 'test'))
fpath = os.path.join(work_dir, 'output_file')
with open(fpath, 'wb') as fout:
fout.write(os.urandom(1024))
copy_files([fpath], os.path.join(work_dir, 'test'))
assert os.path.exists(os.path.join(work_dir, 'test', 'output_file'))
def consolidate_output(job, config, kallisto_output, rsem_output, fastqc_output):
"""
Combines the contents of the outputs into one tarball and places in output directory or s3
:param JobFunctionWrappingJob job: passed automatically by Toil
:param Namespace config: Argparse Namespace object containing argument inputs
:param str kallisto_output: FileStoreID for Kallisto output
:param tuple(str, str) rsem_output: FileStoreIDs for RSEM output
:param str fastqc_output: FileStoreID for FastQC output
"""
job.fileStore.logToMaster('Consolidating input: {}'.format(config.uuid))
work_dir = job.fileStore.getLocalTempDir()
# Retrieve output file paths to consolidate
rsem_tar, hugo_tar, kallisto_tar, fastqc_tar = None, None, None, None
if rsem_output:
rsem_id, hugo_id = rsem_output
rsem_tar = job.fileStore.readGlobalFile(rsem_id, os.path.join(work_dir, 'rsem.tar.gz'))
hugo_tar = job.fileStore.readGlobalFile(hugo_id, os.path.join(work_dir, 'rsem_hugo.tar.gz'))
if kallisto_output:
kallisto_tar = job.fileStore.readGlobalFile(kallisto_output, os.path.join(work_dir, 'kallisto.tar.gz'))
if fastqc_output:
fastqc_tar = job.fileStore.readGlobalFile(fastqc_output, os.path.join(work_dir, 'fastqc.tar.gz'))
# I/O
if not config.paired:
config.uuid = 'SINGLE-END.{}'.format(config.uuid)
out_tar = os.path.join(work_dir, config.uuid + '.tar.gz')
# Consolidate separate tarballs into one as streams (avoids unnecessary untaring)
tar_list = [x for x in [rsem_tar, hugo_tar, kallisto_tar, fastqc_tar] if x is not None]
with tarfile.open(os.path.join(work_dir, out_tar), 'w:gz') as f_out:
for tar in tar_list:
with tarfile.open(tar, 'r') as f_in:
for tarinfo in f_in:
with closing(f_in.extractfile(tarinfo)) as f_in_file:
if tar == rsem_tar:
tarinfo.name = os.path.join(config.uuid, 'RSEM', os.path.basename(tarinfo.name))
elif tar == hugo_tar:
tarinfo.name = os.path.join(config.uuid, 'RSEM', 'Hugo', os.path.basename(tarinfo.name))
elif tar == kallisto_tar:
tarinfo.name = os.path.join(config.uuid, 'Kallisto', os.path.basename(tarinfo.name))
else:
tarinfo.name = os.path.join(config.uuid, 'QC', os.path.basename(tarinfo.name))
f_out.addfile(tarinfo, fileobj=f_in_file)
# Move to output directory
if config.output_dir:
job.fileStore.logToMaster('Moving {} to output dir: {}'.format(config.uuid, config.output_dir))
mkdir_p(config.output_dir)
copy_files(file_paths=[os.path.join(work_dir, config.uuid + '.tar.gz')], output_dir=config.output_dir)
# Upload to S3
if config.s3_output_dir:
job.fileStore.logToMaster('Uploading {} to S3: {}'.format(config.uuid, config.s3_output_dir))
s3am_upload(fpath=out_tar, s3_dir=config.s3_output_dir, num_cores=config.cores)
def consolidate_output(job, config, mutect, pindel, muse):
"""
Combine the contents of separate tarball outputs into one via streaming
:param JobFunctionWrappingJob job: passed automatically by Toil
:param Namespace config: Argparse Namespace object containing argument inputs
:param str mutect: MuTect tarball FileStoreID
:param str pindel: Pindel tarball FileStoreID
:param str muse: MuSe tarball FileStoreID
"""
work_dir = job.fileStore.getLocalTempDir()
mutect_tar, pindel_tar, muse_tar = None, None, None
if mutect:
mutect_tar = job.fileStore.readGlobalFile(mutect, os.path.join(work_dir, 'mutect.tar.gz'))
if pindel:
pindel_tar = job.fileStore.readGlobalFile(pindel, os.path.join(work_dir, 'pindel.tar.gz'))
if muse:
muse_tar = job.fileStore.readGlobalFile(muse, os.path.join(work_dir, 'muse.tar.gz'))
out_tar = os.path.join(work_dir, config.uuid + '.tar.gz')
# Consolidate separate tarballs into one as streams (avoids unnecessary untaring)
tar_list = [x for x in [mutect_tar, pindel_tar, muse_tar] if x is not None]
with tarfile.open(os.path.join(work_dir, out_tar), 'w:gz') as f_out:
for tar in tar_list:
with tarfile.open(tar, 'r') as f_in:
for tarinfo in f_in:
with closing(f_in.extractfile(tarinfo)) as f_in_file:
if tar is mutect_tar:
tarinfo.name = os.path.join(config.uuid, 'mutect', os.path.basename(tarinfo.name))
elif tar is pindel_tar:
tarinfo.name = os.path.join(config.uuid, 'pindel', os.path.basename(tarinfo.name))
else:
tarinfo.name = os.path.join(config.uuid, 'muse', os.path.basename(tarinfo.name))
f_out.addfile(tarinfo, fileobj=f_in_file)
# Move to output location
if urlparse(config.output_dir).scheme == 's3':
job.fileStore.logToMaster('Uploading {} to S3: {}'.format(config.uuid, config.output_dir))
s3am_upload(fpath=out_tar, s3_dir=config.output_dir, num_cores=config.cores)
else:
job.fileStore.logToMaster('Moving {} to output dir: {}'.format(config.uuid, config.output_dir))
mkdir_p(config.output_dir)
copy_files(file_paths=[out_tar], output_dir=config.output_dir)
def rsem_quantification(job, config, star_output):
"""
Unpack STAR results and run RSEM (and saving BAM from STAR)
:param JobFunctionWrappingJob job: passed automatically by Toil
:param Namespace config: Argparse Namespace object containing argument inputs
:param tuple(str, str) star_output: FileStoreIDs from STARs output
:return: FileStoreID results from RSEM postprocess
:rtype: str
"""
work_dir = job.fileStore.getLocalTempDir()
cores = min(16, config.cores)
disk = '2G' if config.ci_test else '40G'
if config.wiggle:
transcriptome_id, sorted_id, wiggle_id = star_output
wiggle_path = os.path.join(work_dir, config.uuid + '.wiggle.bg')
job.fileStore.readGlobalFile(wiggle_id, wiggle_path)
if config.s3_output_dir:
s3am_upload(fpath=wiggle_path, s3_dir=config.s3_output_dir, s3_key_path=config.ssec)
if config.output_idr:
copy_files(file_paths=[wiggle_path], output_dir=config.output_dir)
else:
transcriptome_id, sorted_id = star_output
# Save sorted bam if flag is selected
if config.save_bam:
bam_path = os.path.join(work_dir, config.uuid + '.sorted.bam')
job.fileStore.readGlobalFile(sorted_id, bam_path)
if config.s3_output_dir and config.ssec:
s3am_upload(fpath=bam_path, s3_dir=config.s3_output_dir, s3_key_path=config.ssec)
if config.output_dir:
copy_files(file_paths=[bam_path], output_dir=config.output_dir)
# Declare RSEM and RSEM post-process jobs
rsem_output = job.wrapJobFn(run_rsem, config.cores, transcriptome_id, config.rsem_ref, paired=config.paired,
cores=cores, disk=disk)
rsem_postprocess = job.wrapJobFn(run_rsem_postprocess, config.uuid, rsem_output.rv(0), rsem_output.rv(1))
job.addChild(rsem_output)
rsem_output.addChild(rsem_postprocess)
return rsem_postprocess.rv()
