def main():
# Deal with command line
p = optparse.OptionParser(usage="%prog DIR [DIR...]",
version="%prog "+get_version(),
description="Generate QC report for each directory "
"DIR")
p.add_option('--verify',action='store_true',dest='verify',
help="verify the QC products only (don't write the report)")
opts,args = p.parse_args()
if len(args) < 1:
p.error("Need to supply at least one directory")
# Examine projects i.e. supplied directories
for d in args:
project_name = os.path.basename(d)
dir_path = os.path.abspath(d)
p = AnalysisProject(project_name,dir_path)
print "Project: %s" % p.name
print "-"*(len('Project: ')+len(p.name))
print "%d samples | %d fastqs" % (len(p.samples),len(p.fastqs))
if opts.verify:
if not QCReporter(p).verify():
print "Verification: FAILED"
else:
print "Verification: OK"
else:
qc = QCReporter(p).report()
import optparse
import bcftbx.utils as bcf_utils
from bcftbx.cmdparse import CommandParser
from bcftbx.cmdparse import add_debug_option
from bcftbx.cmdparse import add_no_save_option
from bcftbx.cmdparse import add_dry_run_option
from bcftbx.cmdparse import add_nprocessors_option
from bcftbx.cmdparse import add_runner_option
import auto_process_ngs
import auto_process_ngs.settings
import auto_process_ngs.envmod as envmod
from auto_process_ngs.auto_processor import AutoProcess
from auto_process_ngs.samplesheet_utils import predict_outputs
from auto_process_ngs.utils import paginate
__version__ = auto_process_ngs.get_version()
__settings = auto_process_ngs.settings.Settings()
#######################################################################
# Functions
#######################################################################
# Supporting functions
def add_modulefiles_option(p):
"""
Add a --modulefiles option to an OptionParser option
The values can be accessed via the 'modulefiles' property of the
parser.
try:
scp = applications.general.scp(user,host,f,dest)
print "Running %s" % scp
scp.run_subprocess()
except Exception, ex:
raise Exception("Failed to copy %s to %s: %s" % (f,dirn,ex))
#######################################################################
# Main program
#######################################################################
if __name__ == "__main__":
# Set up command line parser
p = CommandParser(description="Utility for managing processed and analysed Illumina "
"sequence data in ANALYSIS_DIR",
version="%prog "+get_version())
# Add info command
p.add_command('info',help="Get information about ANALYSIS_DIR",
usage="%prog info [OPTIONS] ANALYSIS_DIR",
description="Report information on processed Illumina "
"sequence data in ANALYSIS_DIR.")
add_debug_option(p.parser_for('info'))
# Add copy command
p.add_command('copy',help="Copy fastqs from ANALYSIS_DIR",
usage="%prog copy [OPTIONS] ANALYSIS_DIR DEST_DIR",
description="Copy fastqs from ANALYSIS_DIR to DEST_DIR.")
p.parser_for('copy').add_option('--projects',action='store',dest='projects',default=None,
help="Restrict copying to projects matching the "
"supplied pattern")
p.parser_for('copy').add_option('--fastq-dir',action='store',dest='fastq_dir',default=None,
help="Only copy fastqs from the specified FASTQ_DIR")
# The master toctree document. master_doc = 'index' # General information about the project. project = u'auto_process_ngs' copyright = u'2015, Peter Briggs/University of Manchester' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # sys.path.insert(0, os.path.abspath(os.path.join(os.getcwd(), '..'))) from auto_process_ngs import get_version # # The short X.Y version. version = get_version() # The full version, including alpha/beta/rc tags. release = get_version() # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files.
def main():
# Deal with command line
p = optparse.OptionParser(usage="%prog DIR [DIR...]",
version="%prog "+get_version(),
description="Generate QC report for each directory "
"DIR")
p.add_option('--fastq_dir',
action='store',dest='fastq_dir',default=None,
help="explicitly specify subdirectory of DIRs with "
"Fastq files to run the QC on.")
p.add_option('--qc_dir',action='store',dest='qc_dir',default='qc',
help="explicitly specify QC output directory (nb if "
"supplied then the same QC_DIR will be used for each "
"DIR. Non-absolute paths are assumed to be relative to "
"DIR). Default: 'qc'")
reporting = optparse.OptionGroup(p,'Reporting options')
reporting.add_option('-t','--title',action='store',dest='title',
default=None,
help="title for output QC report")
reporting.add_option('-f','--filename',
action='store',dest='filename',default=None,
help="file name for output HTML QC report (default: "
"<DIR>/<QC_DIR>_report.html)")
reporting.add_option('--zip',action='store_true',
dest='zip',default=False,
help="make ZIP archive for the QC report")
reporting.add_option('--force',action='store_true',
dest='force',default=False,
help="force generation of reports even if "
"verification fails")
p.add_option_group(reporting)
verification = optparse.OptionGroup(p,'Verification options')
verification.add_option('--verify',action='store_true',dest='verify',
help="verify the QC products only (don't "
"write the report)")
verification.add_option('-l','--list-unverified',action='store_true',
dest='list_unverified',default=False,
help="list the Fastqs that failed "
"verification")
p.add_option_group(verification)
opts,args = p.parse_args()
if len(args) < 1:
p.error("Need to supply at least one directory")
# Examine projects i.e. supplied directories
retval = 0
for d in args:
project_name = os.path.basename(d)
dir_path = os.path.abspath(d)
p = AnalysisProject(project_name,dir_path,
fastq_dir=opts.fastq_dir)
print "Project: %s" % p.name
print "Fastqs : %s" % p.fastq_dir
if opts.qc_dir is None:
qc_dir = p.qc_dir
else:
qc_dir = opts.qc_dir
if not os.path.isabs(qc_dir):
qc_dir = os.path.join(p.dirn,qc_dir)
# Warning if there is a mismatch
qc_info = p.qc_info(qc_dir)
if qc_info.fastq_dir is not None and \
os.path.join(p.dirn,qc_info.fastq_dir) != p.fastq_dir:
logger.warning("Stored fastq dir mismatch (%s != %s)" %
(p.fastq_dir,qc_info.fastq_dir))
print "QC output dir: %s" % qc_dir
print "-"*(len('Project: ')+len(p.name))
print "%d samples | %d fastqs" % (len(p.samples),len(p.fastqs))
# Verification step
unverified = verify_qc(p,qc_dir)
if unverified:
if opts.list_unverified:
for fq in unverified:
print fq
print "Verification: FAILED"
retval = 1
if not opts.force:
continue
else:
print "Verification: OK"
if opts.verify:
continue
# Report generation
qc_base = os.path.basename(qc_dir)
if opts.filename is None:
out_file = '%s_report.html' % qc_base
else:
out_file = opts.filename
if not os.path.isabs(out_file):
out_file = os.path.join(p.dirn,out_file)
print "Writing QC report to %s" % out_file
report_html= QCReporter(p).report(qc_dir=qc_dir,
title=opts.title,
filename=out_file)
# Generate ZIP archive
if opts.zip:
report_zip = zip_report(p,qc_dir,report_html)
print "ZIP archive: %s" % report_zip
# Finish with appropriate exit code
sys.exit(retval)
# Installation requirements
install_requires = [
'pillow', 'matplotlib', 'pandas', 'genomics-bcftbx', 'nebulizer'
]
# If we're on ReadTheDocs then we can reduce this
# to a smaller set (to avoid build timeouts)
import os
if os.environ.get('READTHEDOCS') == 'True':
install_requires = []
# Setup for installation etc
from setuptools import setup
import auto_process_ngs
setup(
name="auto_process_ngs",
version=auto_process_ngs.get_version(),
description=
'Automatic processing of NGS sequencing data from Illumina sequencers',
long_description=
"""Utilities to help with the automated processing and management
of sequencing data from Illumina's HiSEQ and MiSEQ platforms""",
url='https://github.com/fls-bioinformatics-core/auto_process_ngs',
maintainer='Peter Briggs',
maintainer_email='*****@*****.**',
packages=[
'auto_process_ngs',
'auto_process_ngs.qc',
],
license='Artistic License',
# Pull in dependencies
install_requires=install_requires,
import logging
from bcftbx.IlluminaData import IlluminaData
from bcftbx.IlluminaData import IlluminaDataError
from bcftbx.IlluminaData import SampleSheet
from bcftbx.IlluminaData import samplesheet_index_sequence
from bcftbx.FASTQFile import FastqIterator
from bcftbx.utils import parse_lanes
from auto_process_ngs.barcodes.analysis import BarcodeCounter
from auto_process_ngs.barcodes.analysis import Reporter
from auto_process_ngs.barcodes.analysis import report_barcodes
from auto_process_ngs.bcl2fastq.utils import make_custom_sample_sheet
from auto_process_ngs.bcl2fastq.utils import check_barcode_collisions
from auto_process_ngs.tenx_genomics_utils import has_10x_indices
from auto_process_ngs import get_version
__version__ = get_version()
#######################################################################
# Functions
#######################################################################
def count_barcodes_bcl2fastq(dirn):
"""
Count the barcodes from bcl2fastq output
"""
try:
unaligned = os.path.basename(dirn.rstrip(os.sep))
dirn = os.path.dirname(os.path.abspath(dirn.rstrip(os.sep)))
illumina_data = IlluminaData(dirn, unaligned_dir=unaligned)
Copyright (C) University of Manchester 2013-16 Peter Briggs
"""
# Hack to acquire all scripts that we want to
# install into 'bin'
from glob import glob
scripts = []
for pattern in ('bin/*.py','bin/*.sh',):
scripts.extend(glob(pattern))
# Setup for installation etc
from setuptools import setup
import auto_process_ngs
setup(name = "auto_process_ngs",
version = auto_process_ngs.get_version(),
description = 'Automatic processing of NGS sequencing data from Illumina sequencers',
long_description = """Utilities to help with the automated processing and management
of sequencing data from Illumina's HiSEQ and MiSEQ platforms""",
url = 'https://github.com/fls-bioinformatics-core/auto_process_ngs',
maintainer = 'Peter Briggs',
maintainer_email = '*****@*****.**',
packages = ['auto_process_ngs',
'auto_process_ngs.qc',],
license = 'Artistic License',
# Pull in dependencies
# See http://stackoverflow.com/questions/19738085/why-isnt-setup-py-dependency-links-doing-anything for info on use of 'dependency_links'
# Note that pip 1.5 needs --process-dependency-links for these
# to work; for pip 1.6 even this will be removed so then you must
# do pip install -r requirements.txt first
install_requires = ['pillow',