def test_subprocess_check_output(self):
"""Command: run command and capture output
"""
cmd = Command('echo','hello')
rc,out = cmd.subprocess_check_output()
self.assertEqual(rc,0)
self.assertEqual(out,"hello\n")
def test_has_exe(self):
"""Command: check 'has_exe' property works
"""
cmd = Command('ls','-l')
self.assertTrue(cmd.has_exe)
cmd = Command('cannot_possibly_exist_3')
self.assertFalse(cmd.has_exe)
def test_make_wrapper_script_handle_args_with_spaces(self):
"""Command: check 'make_wrapper_script' method handles arguments with spaces
"""
cmd = Command('echo','hello world')
self.assertEqual(cmd.make_wrapper_script(),
"echo hello world")
self.assertEqual(cmd.make_wrapper_script(quote_spaces=True),
"echo 'hello world'")
def test_command_extended(self):
"""Command: construct command in multiple calls
"""
cmd = Command('ls')
cmd.add_args('-l','-d')
self.assertEqual(cmd.command,'ls')
self.assertEqual(cmd.args,['-l','-d'])
self.assertEqual(cmd.command_line,['ls','-l','-d'])
self.assertEqual(str(cmd),'ls -l -d')
def info_func(p):
name = os.path.basename(p)
exe = find_program(p)
version = ''
output = Command(exe).subprocess_check_output()[1]
for line in output.split('\n'):
if line.startswith(name):
try:
version = line.split()[1]
except IndexError:
pass
break
return (exe, name.upper(), version)
def test_command_simple(self):
"""Command: construct command in single call
"""
cmd = Command('ls','-l','-d')
self.assertEqual(cmd.command,'ls')
self.assertEqual(cmd.args,['-l','-d'])
self.assertEqual(cmd.command_line,['ls','-l','-d'])
self.assertEqual(str(cmd),'ls -l -d')
def test_command_handles_forward_slash(self):
"""Command: check forward slash is handled correctly
"""
cmd = Command('find','-name','"*"','-exec','grep','"hello"','{}','\\;')
self.assertEqual(cmd.command,'find')
self.assertEqual(cmd.args,['-name','"*"','-exec','grep','"hello"','{}','\\;'])
self.assertEqual(cmd.command_line,['find','-name','"*"','-exec',
'grep','"hello"','{}','\\;'])
self.assertEqual(str(cmd),'find -name "*" -exec grep "hello" {} \\;')
def fastq_screen_tag(conf_file,
fastq_in,
out_dir,
aligner=None,
threads=1,
tempdir=None):
"""
Run 'fastq_screen' and output tagged fastq file
Raises an Exception in the event of an error.
Arguments:
conf_file (str): path to the fastq_screen .conf file
fastq_in (str): path to the FASTQ file to screen
out_dir (str): path to the output directory to put
the tagged FASTQ in
aligner (str): optional, name of the aligner to pass
to fastq_screen (default: don't specify the aligner)
threads (int): optional, the number of threads to
use when running fastq_screen (default: 1)
tempdir (str): optional, directory to create temporary
working directories in when running fastq_screen
Returns:
String: path to the tagged output FASTQ file
"""
# Make a temporary working directory
work_dir = tempfile.mkdtemp(suffix='.fastq_screen', dir=tempdir)
# Build fastq_screen command
fastq_screen_cmd = Command('fastq_screen', '--subset', 0, '--threads',
threads, '--conf', conf_file, '--tag',
'--outdir', work_dir)
if args.aligner is not None:
fastq_screen_cmd.add_args('--aligner', args.aligner)
fastq_screen_cmd.add_args(fastq_in)
print("Running %s" % fastq_screen_cmd)
# Run the command
exit_code = fastq_screen_cmd.run_subprocess(working_dir=work_dir)
if exit_code != 0:
err_msg = "Screening %s against %s failed (exit code %d)" % \
(fastq_in,conf_file,exit_code)
else:
# Handle the outputs
tagged_fastq = os.path.basename(strip_ext(fastq_in,'.fastq')) \
+ '.tagged.fastq'
if not os.path.exists(os.path.join(work_dir, tagged_fastq)):
err_msg = "Failed to generated tagged fastq file %s" % \
tagged_fastq
exit_code = 1
else:
os.rename(os.path.join(work_dir, tagged_fastq),
os.path.join(out_dir, tagged_fastq))
# Clean up working directory
shutil.rmtree(work_dir)
# Raise exception if there was a problem
if exit_code != 0:
raise Exception(err_msg)
# Return path to tagged file
return os.path.join(out_dir, tagged_fastq)
def test_conda_wrapper_activate_command(self):
"""
CondaWrapper: check environment activation command
"""
self._make_mock_conda()
conda = CondaWrapper(conda=self.conda)
self.assertEqual(conda.activate_env_cmd("[email protected]").command_line,
Command(
'source',
os.path.join(self.conda_bin_dir,'activate'),
'[email protected]').command_line)
def test_make_wrapper_script(self):
"""Command: check 'make_wrapper_script' method works
"""
cmd = Command('echo','hello')
self.assertEqual(cmd.make_wrapper_script(),
"echo hello")
self.assertEqual(cmd.make_wrapper_script(shell="/bin/bash"),
"#!/bin/bash\necho hello")
self.assertEqual(
cmd.make_wrapper_script(prologue="# Prints hello"),
"# Prints hello\necho hello")
self.assertEqual(
cmd.make_wrapper_script(epilogue="# End of script"),
"echo hello\n# End of script")
self.assertEqual(
cmd.make_wrapper_script(
shell="/bin/bash",
prologue="echo \"# $(hostname)\"",
epilogue="echo \"# $(date)\""),
"#!/bin/bash\necho \"# $(hostname)\"\n"
"echo hello\necho \"# $(date)\"")
fp = StringIO()
cmd.make_wrapper_script(fp=fp)
self.assertEqual(fp.getvalue(),"echo hello")
def test_run_subprocess(self):
"""Command: run command using subprocess
"""
cmd = Command('ls','-l')
cmd.run_subprocess(log='/dev/null')
# Report number of reads assigned to each sample
samples = well_list.samples()
samples.insert(0, unassigned)
sample_counts_file = os.path.join(tmp_dir, "sample_counts.txt")
report("Number of reads assigned to each sample:")
with open(sample_counts_file, 'w') as fp:
fp.write("#Sample\tNreads\n")
for sample in samples:
count = sample_counts[sample]
report("-- %s: %d" % (sample, count))
fp.write("%s\t%s\n" % (sample, count))
report("Sample counts written to %s" % sample_counts_file)
# Report unassigned barcodes
report("Sorting and counting unassigned barcodes")
cmd = Command('sort')
cmd.add_args(*undetermined_barcode_files)
cmd.add_args('|', 'uniq', '-c', '|', 'sort', '-k', 1, '-n', '-r')
# Run the command via a script
script_file = os.path.join(tmp_dir, "count_undetermined.sh")
cmd.make_wrapper_script(shell='/bin/bash',
filen=script_file,
quote_spaces=True)
log_file = os.path.join(tmp_dir, "count_undetermined.log")
status = Command("/bin/bash",
script_file).run_subprocess(log=log_file,
working_dir=tmp_dir)
os.remove(script_file)
# Check return status
if status != 0:
report("Failed to sort and count undetermined barcodes", fp=sys.stderr)
def main():
# Deal with command line
p = argparse.ArgumentParser(description="Generate QC report for each "
"directory DIR")
p.add_argument('-v',
'--version',
action='version',
version="%(prog)s " + __version__)
p.add_argument('--protocol',
action='store',
dest='qc_protocol',
default=None,
help="explicitly specify QC protocol (must be one of "
"%s). Default is to determine the protocol "
"automatically (recommended)" %
str(','.join(["'%s'" % pr for pr in PROTOCOLS])))
p.add_argument('--qc_dir',
action='store',
dest='qc_dir',
default='qc',
help="explicitly specify QC output directory (nb if "
"supplied then the same QC_DIR will be used for each "
"DIR. Non-absolute paths are assumed to be relative to "
"DIR). Default: 'qc'")
p.add_argument('--fastq_dir',
action='store',
dest='fastq_dir',
default=None,
help="explicitly specify subdirectory of DIRs with "
"Fastq files to run the QC on")
reporting = p.add_argument_group('Reporting options')
reporting.add_argument('-t',
'--title',
action='store',
dest='title',
default=None,
help="title for output QC reports")
reporting.add_argument('-f',
'--filename',
action='store',
dest='filename',
default=None,
help="file name for output HTML QC report "
"(default: <DIR>/<QC_DIR>_report.html)")
reporting.add_argument('--zip',
action='store_true',
dest='zip',
default=False,
help="make ZIP archive for the QC report")
reporting.add_argument('--multiqc',
action='store_true',
dest='multiqc',
default=False,
help="generate MultiQC report")
reporting.add_argument('--force',
action='store_true',
dest='force',
default=False,
help="force generation of reports even if "
"verification fails")
data_dir_group = reporting.add_mutually_exclusive_group()
data_dir_group.add_argument('--data-dir',
action='store_true',
dest='use_data_dir',
help="create a data directory with copies "
"of QC artefacts needed for the HTML "
"report (NB data directory will always "
"be created for multi-project reports, "
"unless --no-data-dir is specified)")
data_dir_group.add_argument('--no-data-dir',
action='store_true',
dest='no_data_dir',
help="don't a data directory with copies "
"of QC artefacts (this is the default "
"except for multi-project reports)")
verification = p.add_argument_group('Verification options')
verification.add_argument('--verify',
action='store_true',
dest='verify',
help="verify the QC products only (don't "
"write the report); returns exit code 0 "
"if QC is verified, 1 if not")
deprecated = p.add_argument_group('Deprecated options')
deprecated.add_argument('-l',
'--list-unverified',
action='store_true',
dest='list_unverified',
default=False,
help="deprecated: does nothing (Fastqs with "
"missing QC outputs can no longer be listed)")
deprecated.add_argument('--strand_stats',
action='store_true',
dest='fastq_strand',
default=False,
help="deprecated: does nothing (strand stats "
"are automatically included if present)")
p.add_argument('dirs',
metavar="DIR",
nargs='+',
help="directory to report QC for; can be a project "
"directory (in which case the default QC directory "
"will be reported), or a QC directory within a "
"project")
args = p.parse_args()
# Report name and version
print("%s version %s" % (os.path.basename(sys.argv[0]), __version__))
# Report arguments
if sys.argv[1:]:
print("\n%s" % ' '.join(
['"%s"' % arg if ' ' in arg else arg for arg in sys.argv[1:]]))
# Report working directory
print("\nCWD %s" % os.getcwd())
# Check for MultiQC if required
if args.multiqc:
if find_program("multiqc") is None:
logging.critical("MultiQC report requested but 'multiqc' "
"not available")
sys.exit(1)
# Get projects and QC dirs from supplied directories
projects = []
for d in args.dirs:
print("\n**** Examining directory %s ****" % d)
# Check if directory is a QC dir
qc_dir = None
# Look for 'qc.info' in current directory
if os.path.exists(os.path.join(os.path.abspath(d), 'qc.info')):
print("...located 'qc.info', assuming this is QC dir")
qc_dir = os.path.abspath(d)
# Locate parent project dir
metadata_file = locate_project_info_file(qc_dir)
if metadata_file is not None:
p = AnalysisProject(os.path.dirname(metadata_file))
print("...located parent project: %s" % p.dirn)
else:
# Unable to locate project directory
print("...failed to locate parent project metadata file")
# Fall back to location of Fastq files
qc_info = AnalysisProjectQCDirInfo(
os.path.join(qc_dir, 'qc.info'))
if qc_info.fastq_dir is not None:
project_dir = os.path.abspath(qc_info.fastq_dir)
if os.path.basename(project_dir).startswith('fastqs'):
# Use the next level up
project_dir = os.path.dirname(project_dir)
print("...putative parent project dir: %s (from "
" Fastq dir)" % project_dir)
p = AnalysisProject(project_dir)
else:
# Failed to locate Fastqs
logging.fatal("Unable to locate parent project")
# Exit with an error
sys.exit(1)
# Issue a warning if a QC dir was explicitly
# specified on the command line
if args.qc_dir is not None:
logging.warning("--qc_dir has been ignored for this "
"directory")
else:
# Assume directory is a project
p = AnalysisProject(os.path.abspath(d))
print("...assuming this is a project dir")
# Identify the QC directory
if args.qc_dir is None:
qc_dir = p.qc_dir
else:
qc_dir = args.qc_dir
if not os.path.isabs(qc_dir):
qc_dir = os.path.join(p.dirn, qc_dir)
print("...QC directory: %s" % qc_dir)
# Explicitly set the QC directory location)
p.use_qc_dir(qc_dir)
# Locate the Fastq dir
qc_info = p.qc_info(qc_dir)
if args.fastq_dir is None:
fastq_dir = qc_info.fastq_dir
if fastq_dir is None:
fastq_dir = p.fastq_dir
else:
fastq_dir = args.fastq_dir
if qc_info.fastq_dir is not None:
if os.path.join(p.dirn, qc_info.fastq_dir) != fastq_dir:
logging.warning("Stored fastq dir mismatch "
"(%s != %s)" %
(fastq_dir, qc_info.fastq_dir))
print("...using Fastqs dir: %s" % p.fastq_dir)
p.use_fastq_dir(fastq_dir, strict=False)
projects.append(p)
# Verify QC for projects
print("\n**** Verifying QC ****")
retval = 0
report_projects = []
for p in projects:
print("\nProject: %s" % p.name)
print("-" * (len('Project: ') + len(p.name)))
print("%d sample%s | %d fastq%s" % (
len(p.samples),
's' if len(p.samples) != 1 else '',
len(p.fastqs),
's' if len(p.fastqs) != 1 else '',
))
# QC metadata
qc_dir = p.qc_dir
qc_info = p.qc_info(qc_dir)
# Set QC protocol for verification
if args.qc_protocol is None:
protocol = qc_info.protocol
if protocol is None:
protocol = determine_qc_protocol(p)
else:
protocol = args.qc_protocol
print("Verifying against QC protocol '%s'" % protocol)
# Verification step
if len(p.fastqs) == 0:
logging.critical("No Fastqs!")
verified = False
else:
try:
verified = verify_project(p, qc_dir, protocol)
except Exception as ex:
logging.critical("Error: %s" % ex)
verified = False
if not verified:
print("Verification: FAILED")
if not args.force:
retval = 1
continue
else:
print("--force specified, ignoring previous errors")
else:
print("Verification: OK")
if args.verify:
continue
report_projects.append(p)
# Generate QC report
if report_projects:
# Set defaults from primary project
p = report_projects[0]
qc_base = os.path.basename(p.qc_dir)
# Filename and location for report
if args.filename is None:
out_file = '%s_report.html' % qc_base
else:
out_file = args.filename
if not os.path.isabs(out_file):
out_file = os.path.join(p.dirn, out_file)
out_dir = os.path.dirname(out_file)
# MultiQC report
if args.multiqc:
multiqc_report = os.path.join(out_dir,
"multi%s_report.html" % qc_base)
# Check if we need to rerun MultiQC
if os.path.exists(multiqc_report) and not args.force:
run_multiqc = False
for p in report_projects:
multiqc_mtime = os.path.getmtime(multiqc_report)
for f in os.listdir(p.qc_dir):
if os.path.getmtime(os.path.join(p.qc_dir,f)) > \
multiqc_mtime:
# Input is newer than report
run_multiqc = True
break
else:
run_multiqc = True
# (Re)run MultiQC
if run_multiqc:
multiqc_cmd = Command('multiqc', '--title', '%s' % args.title,
'--filename', '%s' % multiqc_report,
'--force')
for p in report_projects:
multiqc_cmd.add_args(p.qc_dir)
print("\nRunning %s" % multiqc_cmd)
multiqc_retval = multiqc_cmd.run_subprocess()
if multiqc_retval == 0 and os.path.exists(multiqc_report):
print("MultiQC: %s\n" % multiqc_report)
else:
print("MultiQC: FAILED")
retval += 1
else:
print("MultiQC: %s (already exists)\n" % multiqc_report)
# Create data directory?
use_data_dir = (len(projects) > 1)
if args.use_data_dir:
use_data_dir = True
elif args.no_data_dir:
use_data_dir = False
# Generate report
report_html = report(report_projects,
title=args.title,
filename=out_file,
relative_links=True,
use_data_dir=use_data_dir,
make_zip=args.zip)
print("Wrote QC report to %s" % out_file)
# Finish with appropriate exit code
print("%s completed: exit code %s (%s)" %
(os.path.basename(sys.argv[0]), retval,
('ok' if retval == 0 else 'error')))
sys.exit(retval)