def main():
info(' '.join(sys.argv))
info()
parser = OptionParser(usage='Usage: ' + basename(__file__) + ' --chr chr --vcf VCF_file --samples Sample1,Sample2 '
'--bams BAM_file1,BAM_file2 -o Output_directory '
'--features BED_file')
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser)
parser.add_option('-o', dest='output_dir')
parser.add_option('--samples', dest='sample_names')
parser.add_option('--bams', dest='bams')
parser.add_option('--vcf', dest='vcf_fpath')
parser.add_option('--chr', dest='chrom')
parser.add_option('--features', dest='features', help='BED file with real CDS/Exon/Gene/Transcript regions with '
'annotations (default "features" is in system_config)')
(opts, args) = parser.parse_args(sys.argv[1:])
cnf = Config(opts.__dict__, determine_sys_cnf(opts), {})
cnf.verbose = False
if not cnf.output_dir or not cnf.vcf_fpath or not cnf.chrom:
critical(parser.usage)
cnf.features = cnf.features or cnf.genome.features
samples = [BaseSample(sample_name, None, bam=bam) for (sample_name, bam) in zip(cnf.sample_names.split(','), cnf.bams.split(','))]
split_bams(cnf, samples, cnf.vcf_fpath)
info('Done.')
def proc_args(argv):
info(' '.join(sys.argv))
info()
description = 'This script generates target QC reports for each BAM provided as an input. ' \
'Usage: ' + basename(__file__) + ' sample2bam.tsv --bed target.bed --contols sample1:sample2 -o results_dir'
parser = OptionParser(description=description, usage=description)
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser)
parser.add_option('-o', dest='output_dir', metavar='DIR', default=join(os.getcwd(), 'seq2c'))
parser.add_option('--bed', dest='bed', help='BED file to run Seq2C analysis')
parser.add_option('-c', '--controls', dest='controls', help='Optional control sample names for Seq2C. For multiple controls, separate them using :')
parser.add_option('--seq2c-opts', dest='seq2c_opts', help='Options for the final lr2gene.pl script.')
parser.add_option('--no-prep-bed', dest='prep_bed', help=SUPPRESS_HELP, action='store_false', default=True)
(opts, args) = parser.parse_args()
logger.is_debug = opts.debug
if len(args) == 0:
parser.print_usage()
sys.exit(1)
if len(args) == 1 and not args[0].endswith('.bam'):
sample_names, bam_fpaths = read_samples(verify_file(args[0], is_critical=True, description='Input sample2bam.tsv'))
bam_by_sample = OrderedDict()
for s, b in zip(sample_names, bam_fpaths):
bam_by_sample[s] = b
else:
bam_by_sample = find_bams(args)
run_cnf = determine_run_cnf(opts, is_wgs=not opts.__dict__.get('bed'))
cnf = Config(opts.__dict__, determine_sys_cnf(opts), run_cnf)
check_genome_resources(cnf)
cnf.output_dir = adjust_path(cnf.output_dir)
verify_dir(dirname(cnf.output_dir), is_critical=True)
safe_mkdir(cnf.output_dir)
if not cnf.project_name:
cnf.project_name = basename(cnf.output_dir)
info('Project name: ' + cnf.project_name)
cnf.proc_name = 'Seq2C'
set_up_dirs(cnf)
samples = [
source.TargQC_Sample(name=s_name, dirpath=join(cnf.output_dir, s_name), bam=bam_fpath)
for s_name, bam_fpath in bam_by_sample.items()]
info('Samples: ')
for s in samples:
info(' ' + s.name)
samples.sort(key=lambda _s: _s.key_to_sort())
target_bed = verify_bed(cnf.bed, is_critical=True) if cnf.bed else None
if not cnf.only_summary:
cnf.qsub_runner = adjust_system_path(cnf.qsub_runner)
if not cnf.qsub_runner: critical('Error: qsub-runner is not provided is sys-config.')
verify_file(cnf.qsub_runner, is_critical=True)
return cnf, samples, target_bed, cnf.output_dir
def main():
info(' '.join(sys.argv))
info()
description = 'This script runs preprocessing.'
parser = OptionParser(description=description)
parser.add_option('-1', dest='left_reads_fpath', help='Left reads fpath')
parser.add_option('-2', dest='right_reads_fpath', help='Right reads fpath')
parser.add_option('--sample', dest='sample_name', help='Sample name')
parser.add_option('-o', dest='output_dir', help='Output directory path')
parser.add_option(
'--downsample-to',
dest='downsample_to',
default=None,
type='int',
help=
'Downsample reads to avoid excessive processing times with large files. '
'Default is 1 million. Set to 0 to turn off downsampling.')
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser, threads=1)
(opts, args) = parser.parse_args()
logger.is_debug = opts.debug
cnf = Config(opts.__dict__, determine_sys_cnf(opts),
determine_run_cnf(opts))
left_reads_fpath = verify_file(opts.left_reads_fpath, is_critical=True)
right_reads_fpath = verify_file(opts.right_reads_fpath, is_critical=True)
output_dirpath = adjust_path(
opts.output_dir) if opts.output_dir else critical(
'Please, specify output directory with -o')
verify_dir(dirname(output_dirpath),
description='output_dir',
is_critical=True)
with workdir(cnf):
sample_name = cnf.sample_name
if not sample_name:
sample_name = _get_sample_name(left_reads_fpath, right_reads_fpath)
results_dirpath = run_fastq(cnf,
sample_name,
left_reads_fpath,
right_reads_fpath,
output_dirpath,
downsample_to=cnf.downsample_to)
verify_dir(results_dirpath, is_critical=True)
info()
info('*' * 70)
info('Fastqc results:')
info(' ' + results_dirpath)
def main():
info(' '.join(sys.argv))
info()
description = 'This script runs preprocessing.'
parser = OptionParser(description=description)
parser.add_option('-1', dest='left_reads_fpath', help='Left reads fpath')
parser.add_option('-2', dest='right_reads_fpath', help='Right reads fpath')
parser.add_option('--sample', dest='sample_name', help='Sample name')
parser.add_option('-o', dest='output_dir', help='Output directory path')
parser.add_option('--downsample-to', dest='downsample_to', default=None, type='int',
help='Downsample reads to avoid excessive processing times with large files. '
'Default is 1 million. Set to 0 to turn off downsampling.')
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser, threads=1)
(opts, args) = parser.parse_args()
if not opts.left_reads_fpath or not opts.right_reads_fpath or not opts.output_dir:
parser.print_usage()
verify_file(opts.left_reads_fpath, is_critical=False)
left_reads_fpath = adjust_path(opts.left_reads_fpath)
verify_file(opts.right_reads_fpath, is_critical=False)
right_reads_fpath = adjust_path(opts.right_reads_fpath)
output_dirpath = adjust_path(opts.output_dir) if opts.output_dir else critical('Please, specify output directory with -o')
verify_dir(dirname(output_dirpath), description='output_dir', is_critical=True)
left_reads_fpath, right_reads_fpath, output_dirpath =\
map(_proc_path, [left_reads_fpath, right_reads_fpath, output_dirpath])
ssh = connect_to_server(server_url='blue.usbod.astrazeneca.net', username='******', password='******')
fastqc_py = get_script_cmdline(None, 'python', 'scripts/pre/fastqc.py')
fastqc_py = fastqc_py.replace(REPORTING_SUITE_PATH_CLARITY, REPORTING_SUITE_PATH_WALTHAM)
fastqc_py = fastqc_py.replace(PYTHON_PATH_CLARITY, PYTHON_PATH_WALTHAM)
cmdl = '{fastqc_py} -1 {left_reads_fpath} -2 {right_reads_fpath} -o {output_dirpath}'
if opts.sample_name:
cmdl += ' --sample {opts.sample_name}'
if opts.downsample_to:
cmdl += ' --downsample-to ' + str(int(opts.downsample_to))
cmdl = cmdl.format(**locals())
cmdl += ' 2>&1'
info(cmdl)
stdin, stdout, stderr = ssh.exec_command(cmdl)
for l in stdout:
err(l, ending='')
info()
ssh.close()
def get_args():
description = (
'Plots a Circos plot given vardict variant file (with all dbSNP SNPs, not the PASS one), '
'Seq2C CNV calls and Manta SVs.')
parser = OptionParser(description=description)
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser, threads=1)
parser.add_option('--bed', dest='bed_fpath', help='Path to BED file')
parser.add_option('-v', '--mutations', dest='mutations_fpath', help='Path to VarDict.txt file')
parser.add_option('-c', '--seq2c', dest='seq2c_tsv_fpath', help='Path to seq2c copy number file')
parser.add_option('--sv', dest='sv_fpath', help='Path to Manta SV call vcf.gz file')
parser.add_option('-s', '--sample', dest='sample', help='Identifier of sample in VarDict and Seq2c files')
parser.add_option('-o', '--output-dir', dest='output_dir', default="./",
help='Output directory. Defaults to ./')
(opts, args) = parser.parse_args()
run_cnf = determine_run_cnf(opts)
cnf = Config(opts.__dict__, determine_sys_cnf(opts), run_cnf)
return cnf
def main():
info(' '.join(sys.argv))
info()
description = 'This script converts Vardict TXT file to VCF.'
parser = OptionParser(
description=description,
usage='Usage: ' + basename(__file__) +
' [-o Output_directory -c Var_caller_name] Project_directory')
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser)
parser.add_option('--log-dir', dest='log_dir', default='-')
parser.add_option('-c', '--caller', dest='caller_name', default='vardict')
parser.add_option('-o', dest='output_dir', help='Output directory.')
cnf, bcbio_project_dirpaths, bcbio_cnfs, final_dirpaths, tags, is_wgs_in_bcbio, is_rnaseq \
= process_post_bcbio_args(parser)
if not bcbio_project_dirpaths:
parser.print_help(file=sys.stderr)
sys.exit(1)
bcbio_structures = []
for bcbio_project_dirpath, bcbio_cnf, final_dirpath in zip(
bcbio_project_dirpaths, bcbio_cnfs, final_dirpaths):
bs = BCBioStructure(cnf, bcbio_project_dirpath, bcbio_cnf,
final_dirpath)
bcbio_structures.append(bs)
cnf.work_dir = cnf.work_dir or adjust_path(join(cnf.output_dir, 'work'))
safe_mkdir(cnf.work_dir)
info('')
info('*' * 70)
for bs in bcbio_structures:
for sample in bs.samples:
if sample.phenotype != 'normal':
convert_vardict_txts_to_bcbio_vcfs(cnf, bs, sample)
def proc_opts():
parser = OptionParser()
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser)
parser.add_option('--expose-only',
dest='expose_to_ngs_server_only',
action='store_true',
default=False,
help='Only add project to the webserver')
parser.add_option('--no-expose',
dest='expose',
action='store_false',
default=True,
help='Do not expose the reports')
parser.add_option('-o', dest='output_dir')
parser.add_option('--bed',
dest='bed',
help='BED file to run targetSeq and Seq2C analysis on.')
parser.add_option('--downsample-to', dest='downsample_to', type='int')
(opts, args) = parser.parse_args()
logger.is_debug = opts.debug
if len(args) < 1:
critical('Usage: ' + __file__ + ' *.fq.gz -o output_dir')
# if len(args) < 2:
# info('No dataset path specified, assuming it is the current working directory')
# dataset_dirpath = adjust_path(os.getcwd())
# jira_url = args[0]
fastq_fpaths = [verify_file(fpath) for fpath in args]
fastq_fpaths = [fpath for fpath in fastq_fpaths if fpath]
info(str(len(fastq_fpaths)) + ' fastq files')
run_cnf = determine_run_cnf(opts)
cnf = Config(opts.__dict__, determine_sys_cnf(opts), run_cnf)
cnf.output_dir = adjust_path(cnf.output_dir)
info('Writing to ' + str(cnf.output_dir))
cnf.project_name = cnf.project_name or 'preproc'
if cnf.work_dir:
cnf.debug = True
else:
all_work_dir = join(cnf.output_dir, 'work')
safe_mkdir(all_work_dir)
latest_fpath = join(all_work_dir, 'latest')
if cnf.reuse_intermediate:
cnf.work_dir = latest_fpath
else:
cnf.work_dir = join(
all_work_dir,
datetime.datetime.now().strftime("%Y-%b-%d_%H-%M"))
if islink(latest_fpath):
os.remove(latest_fpath)
if isdir(latest_fpath):
shutil.rmtree(latest_fpath)
if not exists(latest_fpath):
os.symlink(basename(cnf.work_dir), latest_fpath)
cnf.work_dir = adjust_path(cnf.work_dir)
safe_mkdir(cnf.work_dir)
cnf.log_dir = join(cnf.work_dir, 'log')
safe_mkdir(cnf.log_dir)
set_up_log(cnf)
try:
subprocess.call(['chmod', '-R', 'g+w', cnf.work_dir])
except OSError:
err(traceback.format_exc())
pass
if cnf.samplesheet:
cnf.samplesheet = verify_file(cnf.samplesheet, is_critical=True)
info(' '.join(sys.argv))
info()
info('Created a temporary working directory: ' + cnf.work_dir)
if cnf.project_name:
info('Project name: ' + cnf.project_name)
if cnf.samplesheet:
info('Using custom sample sheet ' + cnf.samplesheet)
check_genome_resources(cnf)
check_system_resources(cnf, optional=['fastq'])
return cnf, cnf.output_dir, fastq_fpaths
def main():
info(' '.join(sys.argv))
info()
description = 'This script evaluate capture target.'
parser = OptionParser(description=description)
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser)
parser.add_option('--log-dir', dest='log_dir')
parser.add_option('--exac-only-filtering',
dest='prepare_for_exac',
action='store_true',
default=False,
help='Export filtered regions to ExAC browser.')
parser.add_option('--exac',
dest='add_to_exac',
action='store_true',
default=False,
help='Export coverage data to ExAC browser.')
parser.add_option('--bed',
'--capture',
'--amplicons',
dest='bed',
help='BED file to overlap.')
parser.add_option(
'--tricky-regions',
dest='tricky_regions',
action='store_true',
default=False,
help='Use high GC, low GC, low complexity regions to overlap.')
parser.add_option('--min-percent',
dest='min_percent',
default='0.5',
help='Minimal percent of region which has low coverage.')
parser.add_option(
'--min-ratio',
dest='min_ratio',
default='0.5',
help='Minimal percent of samples which share the same feature.')
parser.add_option('--min-depth',
dest='min_depth',
help='Coverage threshold.')
parser.add_option('--metadata',
dest='metadata',
help='Samples type for each project '
'(plasma, cell_line, ffpe, deepseq, exome, wgs).')
parser.add_option('-o', dest='output_dir', help='Output directory.')
cnf, bcbio_project_dirpaths, bcbio_cnfs, final_dirpaths, tags, is_wgs_in_bcbio, is_rnaseq \
= process_post_bcbio_args(parser)
if not cnf.project_name:
cnf.add_to_exac = False
cnf.project_name = 'CaptureTargetEvaluation'
if cnf.prepare_for_exac:
cnf.output_dir = join(get_exac_dir(cnf), 'coverage', cnf.project_name)
elif cnf.output_dir is None:
cnf.output_dir = join(os.getcwd(), cnf.project_name)
cnf.output_dir = safe_mkdir(adjust_path(cnf.output_dir))
cnf.work_dir = safe_mkdir(join(cnf.output_dir, 'work'))
cnf.log_dir = safe_mkdir(join(cnf.work_dir), 'log')
cnf.min_percent = 1 - float(cnf.min_percent)
cnf.min_ratio = float(cnf.min_ratio)
if cnf.min_depth:
cnf.min_depth = int(cnf.min_depth)
if cnf.metadata:
cov_thresholds = {
'deepseq': 250,
'plasma': 100,
'exome': 20,
'ffpe': 10,
'cell_line': 10,
'wgs': 10
}
cnf.min_depths = [
cov_thresholds[type] for type in cnf.metadata.split(',')
]
if len(bcbio_project_dirpaths) < 1:
critical('Usage: ' + __file__ +
' project_bcbio_path [project_bcbio_path] [-o output_dir]')
info()
info('*' * 70)
safe_mkdir(cnf.output_dir)
if cnf.log_dir:
info('log_dirpath: ' + cnf.log_dir)
safe_mkdir(cnf.log_dir)
set_up_log(cnf, 'evaluate_capture_target', cnf.project_name,
cnf.output_dir)
bcbio_structures = []
for bcbio_project_dirpath, bcbio_cnf, final_dirpath in zip(
bcbio_project_dirpaths, bcbio_cnfs, final_dirpaths):
bs = BCBioStructure(cnf, bcbio_project_dirpath, bcbio_cnf,
final_dirpath)
bcbio_structures.append(bs)
cnf.work_dir = cnf.work_dir or adjust_path(join(cnf.output_dir, 'work'))
safe_mkdir(cnf.work_dir)
info('')
info('*' * 70)
regions_fpath = evaluate_capture(cnf, bcbio_structures)
if cnf.add_to_exac:
if not is_us():
err('Exposing to ExAC browser is available only on US server')
return
output_dirpath = join(get_exac_dir(cnf), 'coverage', cnf.project_name)
safe_mkdir(output_dirpath)
if regions_fpath and regions_fpath != join(output_dirpath,
basename(regions_fpath)):
shutil.copy(regions_fpath,
join(output_dirpath, basename(regions_fpath)))
shutil.copy(regions_fpath + '.tbi',
join(output_dirpath, basename(regions_fpath + '.tbi')))
samples = []
sample_names = [s.name for bs in bcbio_structures for s in bs.samples]
for bs in bcbio_structures:
for sample in bs.samples:
sample.name = get_uniq_sample_key(bs.project_name, sample,
sample_names)
samples.append(sample)
calculate_coverage_use_grid(cnf, samples, output_dirpath)
add_project_to_exac(cnf)
else:
info('Use --exac if you want to export data to ExAC browser')
info('Done.')
def main():
description = 'This script runs reporting suite on the bcbio final directory.'
parser = OptionParser(description=description)
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser)
parser.add_option('--load-mongo',
'--mongo-loader',
dest='load_mongo',
action='store_true',
default=defaults['load_mongo'],
help='Load to Mongo DB')
parser.add_option(
'--datahub-path',
dest='datahub_path',
help=
'DataHub directory path to upload final MAFs and CNV (can be remote).')
parser.add_option(
'--email',
dest='email',
help='E-mail address to send notifications on errors and finished jobs.'
)
parser.add_option('--reannotate',
dest='reannotate',
action='store_true',
default=False,
help='Re-annotate BED file with gene names')
parser.add_option('--extended',
dest='extended',
action='store_true',
default=False,
help='Count flagged regions and missed variants')
parser.add_option('--dedup',
dest='dedup',
action='store_true',
default=False,
help='Count duplicates in coverage metrics')
parser.add_option('--seq2c-opts',
dest='seq2c_opts',
help='Options for the final lr2gene.pl script.')
parser.add_option('--seq2c-controls',
dest='seq2c_controls',
help='Additional controls for Seq2C.')
parser.add_option('--deep-seq',
dest='deep_seq',
action='store_true',
default=False,
help='Use run_info_DeepSeq.yaml')
parser.add_option('--wgs',
dest='is_wgs',
action='store_true',
default=None,
help='Ignore sv_regions and run as WGS')
parser.add_option('--only-summary',
dest='only_summary',
action='store_true',
default=False,
help='Only generate project-level report')
parser.add_option('--jira', dest='jira', help='JIRA case path')
parser.add_option('--bed',
'--capture',
'--amplicons',
dest='bed',
help='BED file to run targetSeq and Seq2C analysis on.')
parser.add_option(
'--exons',
'--exome',
dest='exons',
help='Exons BED file to make targetSeq exon/amplicon regions reports.')
parser.add_option('--no-prep-bed',
dest='prep_bed',
help='do not fix input beds and exons',
action='store_false',
default=True)
parser.add_option('--no-dedup',
dest='no_dedup',
action='store_true',
help=SUPPRESS_HELP)
parser.add_option('-f',
'--freq',
'--min-freq',
dest='min_freq',
type='float',
help='Minimum allele frequency for the filtering.')
parser.add_option('-o',
dest='output_dir',
help='Output directory for report combining.')
parser.add_option('--transcripts',
dest='transcripts_fpath',
help='Transcripts for annotation.')
parser.add_option('--no-bam2bigwig',
dest='no_bam2bigwig',
action='store_true',
default=False,
help=SUPPRESS_HELP)
cnf, bcbio_project_dirpaths, bcbio_cnfs, final_dirpaths, tags, is_wgs_in_bcbio, is_rnaseq \
= process_post_bcbio_args(parser)
is_wgs = cnf.is_wgs = cnf.is_wgs or is_wgs_in_bcbio
cnf.run_date = time.localtime()
cnf_project_name = cnf.project_name
if len(bcbio_project_dirpaths) > 1:
cnf.project_name = None
info()
info('*' * 70)
bcbio_structures = []
for bcbio_project_dirpath, bcbio_cnf, final_dirpath in zip(
bcbio_project_dirpaths, bcbio_cnfs, final_dirpaths):
bs = BCBioStructure(cnf,
bcbio_project_dirpath,
bcbio_cnf,
final_dirpath,
is_wgs=is_wgs,
is_rnaseq=is_rnaseq)
bcbio_structures.append(bs)
# Post-processing one bcbio project as usually
if len(bcbio_structures) == 1:
if cnf.min_freq is not None:
info('Min freq for filtering is %f' % cnf.min_freq)
if cnf.steps and cnf.load_mongo and 'MongoLoader' not in cnf.steps:
cnf.steps.append('MongoLoader')
check_system_resources(cnf, required=['qsub'])
bcbio_structure = bcbio_structures[0]
bcbio_runner = BCBioRunner(cnf, bcbio_structure, cnf.bcbio_cnf)
bcbio_runner.post_jobs()
# Special case: multiple projects in input. No post-processing them, but rather combining summary reports together.
elif len(bcbio_structures) > 1:
if cnf_project_name:
cnf.project_name = cnf_project_name
else:
cnf.project_name = '_'.join(
[bs.project_name for bs in bcbio_structures])
if not cnf.output_dir:
cnf.output_dir = join(os.getcwd(), cnf.project_name)
safe_mkdir(cnf.output_dir)
cnf.log_dir = join(cnf.output_dir, 'log')
info('log_dirpath: ' + cnf.log_dir)
safe_mkdir(cnf.log_dir)
set_up_log(cnf, 'miltiple_projects', cnf.project_name, cnf.output_dir)
cnf.work_dir = adjust_path(join(cnf.output_dir, 'work'))
safe_mkdir(cnf.work_dir)
safe_mkdir(adjust_path(join(cnf.output_dir, 'config')))
combine_projects(cnf, bcbio_structures, tags)
def main():
info(' '.join(sys.argv))
info()
description = 'This script makes paired WGS-target clincial reports based on 2 bcbio projects.'
parser = OptionParser(description=description)
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser, threads=1)
parser.add_option(
'--email',
dest='email',
help='E-mail address to send notifications on errors and finished jobs.'
)
parser.add_option('--jira', dest='jira', help='JIRA case path')
parser.add_option('--bed',
'--capture',
'--amplicons',
dest='bed',
help='BED file to run targetSeq and Seq2C analysis on.')
parser.add_option(
'--exons',
'--exome',
dest='exons',
help='Exons BED file to make targetSeq exon/amplicon regions reports.')
parser.add_option('-o',
dest='output_dir',
help='Output directory for report combining.')
cnf, bcbio_project_dirpaths, bcbio_cnfs, final_dirpaths, tags, is_wgs_in_bcbio, is_rnaseq \
= process_post_bcbio_args(parser)
is_wgs = cnf.is_wgs = cnf.is_wgs or is_wgs_in_bcbio
if len(bcbio_project_dirpaths) < 2 or len(bcbio_project_dirpaths) > 2:
critical('Usage: ' + __file__ + ' wgs_project_project_bcbio_path '
'targetseq_project_bcbio_path [-o output_dir]')
info()
info('*' * 70)
bcbio_structures = []
for bcbio_project_dirpath, bcbio_cnf, final_dirpath in zip(
bcbio_project_dirpaths, bcbio_cnfs, final_dirpaths):
bs = BCBioStructure(cnf,
bcbio_project_dirpath,
bcbio_cnf,
final_dirpath,
is_wgs=is_wgs,
is_rnaseq=is_rnaseq)
bcbio_structures.append(bs)
trg_bs = next((bs for bs in bcbio_structures if bs.bed), None)
wgs_bs = next((bs for bs in bcbio_structures if not bs.bed), None)
if not trg_bs and not wgs_bs:
critical('One of the projects must be targeted, and one must be WGS')
if not trg_bs:
critical('One of the projects must be targeted.')
if not wgs_bs:
critical('One of the projects must be WGS.')
if not cnf.project_name:
cnf.project_name = wgs_bs.project_name.replace('_WGS',
'').replace('WGS', '')
if cnf.output_dir is None:
cnf.output_dir = join(os.getcwd(), cnf.project_name)
safe_mkdir(cnf.output_dir)
cnf.log_dir = join(cnf.output_dir, 'log')
info('log_dirpath: ' + cnf.log_dir)
safe_mkdir(cnf.log_dir)
set_up_log(cnf, 'clinical_target2wgs', cnf.project_name, cnf.output_dir)
cnf.work_dir = cnf.work_dir or adjust_path(join(cnf.output_dir, 'work'))
safe_mkdir(cnf.work_dir)
shared_sample_names = set(s.name for s in wgs_bs.samples) & set(
s.name for s in trg_bs.samples)
if not shared_sample_names:
critical('Not shared samples in target and WGS projects.\n'
'Target: ' + ', '.join(s.name for s in trg_bs.samples) +
'WGS: ' + ', '.join(s.name for s in wgs_bs.samples))
info('Shared samples: ' + ', '.join(shared_sample_names))
info('')
info('*' * 70)
run_clinical_target2wgs(cnf.genome, wgs_bs, trg_bs, shared_sample_names,
cnf.output_dir)
def get_args():
info(' '.join(sys.argv))
info()
description = (
'The program will filter the VarDict output after vcf2txt.pl to '
'candidate interpretable mutations, somatic or germline.')
parser = OptionParser(description=description)
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser, threads=1)
parser.add_option('-o', dest='output_file')
parser.add_option('--o-all-transcripts',
dest='all_transcripts_output_file')
parser.add_option('--o-fm', dest='fm_output_file')
parser.add_option('--o-reject', dest='rejected_output_file')
parser.add_option('--cohort-freqs', dest='cohort_freqs_fpath')
parser.add_option('--transcripts', dest='transcripts_fpath')
parser.add_option('-D',
'--min-depth',
dest='filt_depth',
type='int',
help='The minimum total depth')
parser.add_option('-V',
'--min-vd',
dest='min_vd',
type='int',
help='The minimum reads supporting variant')
parser.add_option(
'--gmaf',
dest='min_gmaf',
type='float',
help=
'When the GMAF is greater than specified, it\'s considered common SNP and filtered out.'
)
parser.add_option(
'-f',
'--min-freq',
dest='min_freq',
type='float',
help='The minimum allele frequency for regular variants.')
parser.add_option(
'-F',
'--min-freq-hs',
'--act-min-freq',
dest='act_min_freq',
type='float',
help=
'The minimum allele frequency hotspot somatic mutations, typically lower then -f. '
'Default: 0.01 or half -f, whichever is less')
parser.add_option(
'-N',
'--keep-utr-intronic',
dest='keep_utr_intronic',
action='store_true',
help=
'Keep all intronic and UTR in the output, but will be set as "unknown".'
)
parser.add_option(
'-p',
'--platform',
dest='platform',
help=
'The platform, such as WXS, WGS, RNA-Seq, VALIDATION, etc. No Default. '
'Used for output in FM\'s format')
parser.set_usage('Usage: ' + __file__ +
' vcf2txt_res_fpath [opts] -o output_fpath')
(opts, args) = parser.parse_args()
if len(args) < 1:
critical('Provide the first argument - output from vcf2txt.pl')
logger.is_debug = opts.debug
vcf2txt_res_fpath = verify_file(args[0], is_critical=True)
run_cnf = determine_run_cnf(opts)
cnf = Config(opts.__dict__, determine_sys_cnf(opts), run_cnf)
if not cnf.genome:
critical('Please, specify the --genome option (e.g. --genome hg19)')
check_genome_resources(cnf)
if not cnf.output_file:
critical('Please, specify the output fpath with -o')
info()
return cnf, vcf2txt_res_fpath
def main():
info(' '.join(sys.argv))
info()
description = 'This script makes clinical reports based on multiple bcbio projects.'
parser = OptionParser(description=description)
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser)
parser.add_option('--log-dir', dest='log_dir')
parser.add_option(
'--email',
dest='email',
help='E-mail address to send notifications on errors and finished jobs.'
)
parser.add_option('--metadata',
dest='metadata_csv',
help='CSV file with parameters of each sample.')
parser.add_option('-o',
dest='output_dir',
help='Output directory for report combining.')
cnf, bcbio_project_dirpaths, bcbio_cnfs, final_dirpaths, tags, is_wgs_in_bcbio, is_rnaseq \
= process_post_bcbio_args(parser)
is_wgs = cnf.is_wgs = cnf.is_wgs or is_wgs_in_bcbio
if not cnf.metadata_csv:
critical(
'Provide the path to CSV file with information of each sample')
critical(
'Usage: ' + __file__ +
' project_bcbio_path [project_bcbio_path] --metadata metadata_path [-o output_dir]'
)
cnf.sample_names = []
parameters_info, samples_data = parse_samples_metadata(
cnf, cnf.metadata_csv)
info()
info('*' * 70)
bcbio_structures = []
for bcbio_project_dirpath, bcbio_cnf, final_dirpath in zip(
bcbio_project_dirpaths, bcbio_cnfs, final_dirpaths):
bs = BCBioStructure(cnf,
bcbio_project_dirpath,
bcbio_cnf,
final_dirpath,
is_wgs=is_wgs,
is_rnaseq=is_rnaseq)
bcbio_structures.append(bs)
for s in bs.samples:
assert s.targetcov_json_fpath, str(s.dirpath) + ' ' + str(
s.targqc_dirpath)
if cnf.output_dir is None and cnf.project_name is None:
critical(
'Either -o (output dir) or --project (project name) has to be specified'
)
if not cnf.output_dir:
cnf.output_dir = join(os.getcwd(), cnf.project_name)
if not cnf.project_name:
cnf.project_name = 'Combined_project'
cnf.output_dir = safe_mkdir(adjust_path(cnf.output_dir))
cnf.log_dir = join(cnf.output_dir, 'log')
info('log_dirpath: ' + cnf.log_dir)
safe_mkdir(cnf.log_dir)
set_up_log(cnf, 'combine_clin_reports', cnf.project_name, cnf.output_dir)
cnf.work_dir = cnf.work_dir or adjust_path(join(cnf.output_dir, 'work'))
safe_mkdir(cnf.work_dir)
# shared_sample_names = set(s.name for bs in bcbio_structures for s in bs.samples)
# if not shared_sample_names:
# sample_names = [bs.project_name + ': ' + ', '.join(s.name for s in bs.samples) for bs in bcbio_structures]
# critical('Not shared samples in projects.\n' + '\n'.join(sample_names))
# info('Shared samples: ' + ', '.join(shared_sample_names))
info('')
info('*' * 70)
run_combine_clinical_reports(cnf, bcbio_structures, parameters_info,
samples_data)
def main():
info(' '.join(sys.argv))
info()
description = 'This script generates target QC reports for each BAM provided as an input.'
parser = OptionParser(description=description)
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser, threads=1)
parser.add_option('--work-dir', dest='work_dir', metavar='DIR')
parser.add_option('--log-dir', dest='log_dir')
parser.add_option('--only-summary',
dest='only_summary',
action='store_true')
parser.add_option('-o',
dest='output_dir',
metavar='DIR',
default=join(os.getcwd(), 'targetqc'))
parser.add_option('--reannotate',
dest='reannotate',
action='store_true',
default=False,
help='re-annotate BED file with gene names')
parser.add_option('--dedup',
dest='dedup',
action='store_true',
default=False,
help='count duplicates in coverage metrics')
parser.add_option('--bed',
dest='bed',
help='BED file to run targetSeq and Seq2C analysis on.')
parser.add_option(
'--exons',
'--exome',
'--features',
dest='features',
help=
'Annotated CDS/Exon/Gene/Transcripts BED file to make targetSeq exon/amplicon regions reports.'
)
(opts, args) = parser.parse_args()
logger.is_debug = opts.debug
if len(args) == 0:
critical('No BAMs provided to input.')
bam_fpaths = list(set([abspath(a) for a in args]))
bad_bam_fpaths = []
for fpath in bam_fpaths:
if not verify_bam(fpath):
bad_bam_fpaths.append(fpath)
if bad_bam_fpaths:
critical('BAM files cannot be found, empty or not BAMs:' +
', '.join(bad_bam_fpaths))
run_cnf = determine_run_cnf(opts, is_wgs=not opts.__dict__.get('bed'))
cnf = Config(opts.__dict__, determine_sys_cnf(opts), run_cnf)
if not cnf.project_name:
cnf.project_name = basename(cnf.output_dir)
info('Project name: ' + cnf.project_name)
cnf.proc_name = 'TargQC'
set_up_dirs(cnf)
# cnf.name = 'TargQC_' + cnf.project_name
check_genome_resources(cnf)
verify_bed(cnf.bed, is_critical=True)
bed_fpath = adjust_path(cnf.bed)
info('Using amplicons/capture panel ' + bed_fpath)
features_bed_fpath = adjust_path(
cnf.features) if cnf.features else adjust_path(cnf.genome.features)
info('Features: ' + features_bed_fpath)
genes_fpath = None
if cnf.genes:
genes_fpath = adjust_path(cnf.genes)
info('Custom genes list: ' + genes_fpath)
if not cnf.only_summary:
cnf.qsub_runner = adjust_system_path(cnf.qsub_runner)
if not cnf.qsub_runner:
critical('Error: qsub-runner is not provided is sys-config.')
verify_file(cnf.qsub_runner, is_critical=True)
info('*' * 70)
info()
targqc_html_fpath = run_targqc(cnf, cnf.output_dir, bam_fpaths, bed_fpath,
features_bed_fpath, genes_fpath)
if targqc_html_fpath:
send_email(
cnf, 'TargQC report for ' + cnf.project_name + ':\n ' +
targqc_html_fpath)
def proc_args(argv):
info(' '.join(sys.argv))
info()
description = 'This script generates target QC reports for each BAM provided as an input.'
parser = OptionParser(description=description)
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser)
parser.add_option('--log-dir', dest='log_dir')
parser.add_option('--is-wgs',
dest='is_wgs',
action='store_true',
default=False,
help='whole genome sequencing')
parser.add_option('--is-deep-seq',
dest='is_deep_seq',
action='store_true',
default=False,
help='deep targeted sequencing')
parser.add_option('--only-summary',
dest='only_summary',
action='store_true')
parser.add_option('-o',
dest='output_dir',
metavar='DIR',
default=join(os.getcwd(), 'targetqc'))
parser.add_option('-c', '--caller', dest='caller')
parser.add_option('--qc', dest='qc', action='store_true', default=False)
parser.add_option('--no-qc',
dest='qc',
action='store_false',
default=False)
parser.add_option('--qc-caption', dest='qc_caption', help=SUPPRESS_HELP)
parser.add_option('--no-tsv',
dest='tsv',
action='store_false',
default=True,
help=SUPPRESS_HELP)
(opts, args) = parser.parse_args()
logger.is_debug = opts.debug
if len(args) == 0:
critical('No vcf files provided to input.')
run_cnf = determine_run_cnf(opts,
is_targetseq=opts.is_deep_seq,
is_wgs=opts.is_wgs)
cnf = Config(opts.__dict__, determine_sys_cnf(opts), run_cnf)
vcf_fpath_by_sample = read_samples(args, cnf.caller)
info()
if not cnf.project_name:
cnf.project_name = basename(cnf.output_dir)
info('Project name: ' + cnf.project_name)
cnf.proc_name = 'Variants'
set_up_dirs(cnf)
# cnf.name = 'TargQC_' + cnf.project_name
info(' '.join(sys.argv))
samples = [
source.VarSample(s_name, join(cnf.output_dir, s_name), vcf=vcf_fpath)
for s_name, vcf_fpath in vcf_fpath_by_sample.items()
]
samples.sort(key=lambda _s: _s.key_to_sort())
check_genome_resources(cnf)
if not cnf.only_summary:
cnf.qsub_runner = adjust_system_path(cnf.qsub_runner)
if not cnf.qsub_runner:
critical('Error: qsub-runner is not provided is sys-config.')
verify_file(cnf.qsub_runner, is_critical=True)
return cnf, samples
def main():
info(' '.join(sys.argv))
info()
description = 'This script prepare data for ExAC browser'
parser = OptionParser(description=description)
add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser)
parser.add_option('--log-dir', dest='log_dir', default='-')
parser.add_option('--bed', dest='bed', help='BED file.')
parser.add_option('--evaluate-capture-target',
dest='do_evaluate_capture',
action='store_true',
help='Evaluate capture target.')
parser.add_option('-o',
dest='output_dir',
help='Output directory with ExAC data.')
cnf, bcbio_project_dirpaths, bcbio_cnfs, final_dirpaths, tags, is_wgs_in_bcbio, is_rnaseq \
= process_post_bcbio_args(parser)
if not cnf.genome:
critical(
'Usage: ' + __file__ +
' -g hg19 project_bcbio_path [project_bcbio_path] [--bed bed_fpath] [-o output_dir] [--evaluate-capture-target]'
)
cnf.output_dir = get_exac_dir(cnf)
# if not cnf.output_dir:
# critical('Error! Please specify ExAC browser data directory')
if len(bcbio_project_dirpaths) < 1:
critical(
'Usage: ' + __file__ +
' -g hg19 project_bcbio_path [project_bcbio_path] [--bed bed_fpath] [-o output_dir] [--evaluate-capture-target]'
)
info()
info('*' * 70)
bcbio_structures = []
project_name = cnf.project_name
cnf.project_name = None
for bcbio_project_dirpath, bcbio_cnf, final_dirpath in zip(
bcbio_project_dirpaths, bcbio_cnfs, final_dirpaths):
bs = BCBioStructure(cnf, bcbio_project_dirpath, bcbio_cnf,
final_dirpath)
bcbio_structures.append(bs)
cnf.project_name = project_name
if not cnf.project_name:
if len(bcbio_structures) == 1:
cnf.project_name = bcbio_structures[0].project_name
else:
critical(
'If you combine multiple BCBIO projects you should specify new project name'
)
cnf.caller_name = 'vardict'
if cnf.output_dir is None:
critical('Please specify path to ExAC data directory.')
safe_mkdir(cnf.output_dir)
cnf.log_dir = join(cnf.output_dir, cnf.project_name + '_log')
info('log_dirpath: ' + cnf.log_dir)
safe_mkdir(cnf.log_dir)
set_up_log(cnf, 'prepare_for_exac', cnf.project_name, cnf.output_dir)
cnf.work_dir = cnf.work_dir or adjust_path(
join(cnf.output_dir, 'work', cnf.project_name))
safe_mkdir(cnf.work_dir)
samples = []
for bs in bcbio_structures:
for sample in bs.samples:
sample.name = get_uniq_sample_key(bs.project_name, sample)
samples.append(sample)
info()
info('Preparing variants data')
variants_dirpath = join(cnf.output_dir, 'vardict')
safe_mkdir(variants_dirpath)
combined_vcf_raw_fpath = join(variants_dirpath, cnf.project_name + '.vcf')
combined_vcf_fpath = combined_vcf_raw_fpath + '.gz'
if not cnf.reuse_intermediate or not verify_file(combined_vcf_fpath):
vcf_fpath_by_sname = dict()
for bs in bcbio_structures:
pass_mut_fpaths = get_mutations_fpaths(bs)
vcf_fpaths, pass_vcf_fpaths = convert_vardict_txts_to_bcbio_vcfs(
cnf.work_dir,
cnf.genome.name,
pass_mut_fpaths,
bs.samples,
cnf.caller_name,
output_dirpath=cnf.work_dir,
pass_only=False,
bed_fpath=bs.sv_bed,
min_freq=bs.cnf.variant_filtering['min_freq'],
act_min_freq=bs.cnf.variant_filtering['act_min_freq'])
if not vcf_fpaths and not pass_vcf_fpaths:
continue
for sample, vcf_fpath, pass_vcf_fpath in zip(
bs.samples, vcf_fpaths, pass_vcf_fpaths):
if vcf_fpath and verify_file(vcf_fpath):
vcf_fpath_by_sname[sample.name] = vcf_fpath
elif pass_vcf_fpath and verify_file(pass_vcf_fpath):
vcf_fpath_by_sname[sample.name] = pass_vcf_fpath
if not vcf_fpath_by_sname:
info('No VCFs found, skipping preparing variants')
else:
info()
combined_vcf_fpath = merge_vcfs(cnf, vcf_fpath_by_sname,
combined_vcf_raw_fpath)
project_vcf_dirpath = join(variants_dirpath, cnf.project_name)
safe_mkdir(project_vcf_dirpath)
for sample_name, vcf_fpath in vcf_fpath_by_sname.items():
if verify_file(vcf_fpath) and not verify_file(join(
project_vcf_dirpath, basename(vcf_fpath)),
silent=True):
shutil.move(vcf_fpath, project_vcf_dirpath)
shutil.move(vcf_fpath + '.tbi', project_vcf_dirpath)
info()
info('Saving coverage')
project_cov_dirpath = join(cnf.output_dir, 'coverage', cnf.project_name)
safe_mkdir(project_cov_dirpath)
calculate_coverage_use_grid(cnf, samples, project_cov_dirpath)
if cnf.do_evaluate_capture:
evaluate_capture(cnf, bcbio_project_dirpaths)
if combined_vcf_fpath:
info()
info('Creating BAM files for IGV')
exac_features_fpath = os.path.join(exac_data_dir, cnf.genome.name,
'all_features.bed.gz')
split_bam_files_use_grid(cnf, samples, combined_vcf_fpath,
exac_features_fpath)
else:
warn(
'Combined VCF file does not exist. BAM files for IGV cannot be created'
)
info()
add_project_to_exac(cnf)
info('Done.')