def main(): cnf = read_opts_and_cnfs( description='Plotting Seq2C results.', extra_opts=[ (['--seq2c-results'], dict( dest='seq2c_tsv_fpath') ), (['--key-genes'], dict( dest='key_genes_fpath') ), ], required_keys=['seq2c_tsv_fpath', 'output_dir'], file_keys=['seq2c_tsv_fpath', 'key_genes'], key_for_sample_name=None, ) check_system_resources(cnf) check_genome_resources(cnf) key_gene_names = None if cnf.key_genes_fpath: with open(cnf.key_genes_fpath) as f: key_gene_names = set([l.strip() for l in f.readlines() if l.strip() != '']) plot_fpath = draw_seq2c_plot(cnf, cnf.seq2c_tsv_fpath, cnf.sample, cnf.output_dir, key_gene_names) if plot_fpath: info('Saved plot to ' + plot_fpath)
def main(args): cnf = read_opts_and_cnfs( extra_opts=[ (['--vcf', '--var'], dict( dest='vcf', help='variants to annotate') ), (['--bam'], dict( dest='bam', help='(outdated) used to generate some annotations by GATK') ), (['--match-normal-sample-name'], dict( dest='match_normal_normal_name') ), (['--clinical-reporting'], dict( dest='clinical_reporting', help='used to generate some annotations by GATK', action='store_true', default=None) ), (['--qc'], dict( dest='qc', action='store_true', default=True, help=SUPPRESS_HELP) ), (['--no-qc'], dict( dest='qc', action='store_false', help=SUPPRESS_HELP) ), ], required_keys=['vcf'], file_keys=['bam', 'vcf'], key_for_sample_name='vcf', proc_name=source.varannotate_name) check_system_resources(cnf, required=['java', 'perl', 'snpeff'], optional=['transcripts_fpath']) check_genome_resources(cnf) # info('Using variants ' + cnf['vcf']) # info('Using alignement ' + cnf['bam']) run_one(cnf, process_one, finalize_one) if not cnf['keep_intermediate']: shutil.rmtree(cnf['work_dir'])
def main(): cnf = read_opts_and_cnfs( extra_opts=[ (['--targqc-dir'], dict(dest='targqc_dirpath', )), (['--mutations'], dict(dest='mutations_fpath', )), (['--sv'], dict(dest='sv_fpath', )), (['--sv-vcf'], dict(dest='sv_vcf_fpath', )), (['--varqc'], dict(dest='varqc_json_fpath', )), (['--varqc-after'], dict(dest='varqc_after_json_fpath', )), (['--target-type'], dict( dest='target_type', default='panel', )), (['--bed'], dict(dest='bed_fpath', )), (['--seq2c'], dict(dest='seq2c_tsv_fpath', )), (['--project-level-report'], dict(dest='project_report_path', )), (['--targqc-html'], dict(dest='targqc_report_path', )), (['--match'], dict(dest='match_sample_name', )), (['--jira'], dict(dest='jira_url', )), ], key_for_sample_name=None, required_keys=[], file_keys=[ 'mutations_fpath', 'varqc_json_fpath', 'varqc_after_json_fpath', 'bed_fpath', 'seq2c_tsv_fpath', 'sv_fpath', 'sv_vcf_fpath', #'project_report_path', # DO NOT UNCOMMENT! Project level report might not yet exist ], # do not check mutations_fpath! could be either of: # vardict.PASS.txt, # vardict-java.PASS.txt, # vardict.single.PASS.txt, # vardict.paired.PASS.txt, dir_keys=['targqc_dirpath'], ) check_genome_resources(cnf) check_system_resources(cnf, required=['bedtools'], optional=[]) clin_info = clinical_sample_info_from_cnf(cnf) html_fpath = make_clinical_report(cnf, clin_info, clin_info.sample.clinical_html) info('Clinical report: ' + html_fpath) if not cnf['keep_intermediate']: shutil.rmtree(cnf['work_dir'])
def main(args): cnf, bcbio_structure = bcbio_summary_script_proc_params( BCBioStructure.targqc_name, BCBioStructure.targqc_summary_dir, extra_opts=[ (['--mutations'], dict(dest='mutations_fpath', )), (['--bed', '--capture', '--amplicons'], dict(dest='bed', help='a BED file for capture panel or amplicons')), ]) check_system_resources(cnf, required=['bedtools'], optional=[]) process_all(cnf, bcbio_structure) if not cnf['keep_intermediate']: shutil.rmtree(cnf['work_dir'])
def main(args): cnf = read_opts_and_cnfs( extra_opts=[ (['--var', '--vcf'], dict( dest='vcf', help='variants to evaluate') ), ], required_keys=['vcf'], file_keys=['vcf'], key_for_sample_name='vcf', proc_name=source.varqc_name, ) check_system_resources(cnf) check_genome_resources(cnf) info('Using variants ' + cnf['vcf']) run_one(cnf, process_one, finalize_one) if not cnf['keep_intermediate']: shutil.rmtree(cnf['work_dir'])
def proc_opts(): parser = OptionParser() add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser) parser.add_option('--expose-only', dest='expose_to_ngs_server_only', action='store_true', default=False, help='Only add project to the webserver') parser.add_option('--no-expose', dest='expose', action='store_false', default=True, help='Do not expose the reports') parser.add_option('-o', dest='output_dir') parser.add_option('--bed', dest='bed', help='BED file to run targetSeq and Seq2C analysis on.') parser.add_option('--downsample-to', dest='downsample_to', type='int') (opts, args) = parser.parse_args() logger.is_debug = opts.debug if len(args) < 1: critical('Usage: ' + __file__ + ' *.fq.gz -o output_dir') # if len(args) < 2: # info('No dataset path specified, assuming it is the current working directory') # dataset_dirpath = adjust_path(os.getcwd()) # jira_url = args[0] fastq_fpaths = [verify_file(fpath) for fpath in args] fastq_fpaths = [fpath for fpath in fastq_fpaths if fpath] info(str(len(fastq_fpaths)) + ' fastq files') run_cnf = determine_run_cnf(opts) cnf = Config(opts.__dict__, determine_sys_cnf(opts), run_cnf) cnf.output_dir = adjust_path(cnf.output_dir) info('Writing to ' + str(cnf.output_dir)) cnf.project_name = cnf.project_name or 'preproc' if cnf.work_dir: cnf.debug = True else: all_work_dir = join(cnf.output_dir, 'work') safe_mkdir(all_work_dir) latest_fpath = join(all_work_dir, 'latest') if cnf.reuse_intermediate: cnf.work_dir = latest_fpath else: cnf.work_dir = join( all_work_dir, datetime.datetime.now().strftime("%Y-%b-%d_%H-%M")) if islink(latest_fpath): os.remove(latest_fpath) if isdir(latest_fpath): shutil.rmtree(latest_fpath) if not exists(latest_fpath): os.symlink(basename(cnf.work_dir), latest_fpath) cnf.work_dir = adjust_path(cnf.work_dir) safe_mkdir(cnf.work_dir) cnf.log_dir = join(cnf.work_dir, 'log') safe_mkdir(cnf.log_dir) set_up_log(cnf) try: subprocess.call(['chmod', '-R', 'g+w', cnf.work_dir]) except OSError: err(traceback.format_exc()) pass if cnf.samplesheet: cnf.samplesheet = verify_file(cnf.samplesheet, is_critical=True) info(' '.join(sys.argv)) info() info('Created a temporary working directory: ' + cnf.work_dir) if cnf.project_name: info('Project name: ' + cnf.project_name) if cnf.samplesheet: info('Using custom sample sheet ' + cnf.samplesheet) check_genome_resources(cnf) check_system_resources(cnf, optional=['fastq']) return cnf, cnf.output_dir, fastq_fpaths
def main(): description = 'This script runs reporting suite on the bcbio final directory.' parser = OptionParser(description=description) add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser) parser.add_option('--load-mongo', '--mongo-loader', dest='load_mongo', action='store_true', default=defaults['load_mongo'], help='Load to Mongo DB') parser.add_option( '--datahub-path', dest='datahub_path', help= 'DataHub directory path to upload final MAFs and CNV (can be remote).') parser.add_option( '--email', dest='email', help='E-mail address to send notifications on errors and finished jobs.' ) parser.add_option('--reannotate', dest='reannotate', action='store_true', default=False, help='Re-annotate BED file with gene names') parser.add_option('--extended', dest='extended', action='store_true', default=False, help='Count flagged regions and missed variants') parser.add_option('--dedup', dest='dedup', action='store_true', default=False, help='Count duplicates in coverage metrics') parser.add_option('--seq2c-opts', dest='seq2c_opts', help='Options for the final lr2gene.pl script.') parser.add_option('--seq2c-controls', dest='seq2c_controls', help='Additional controls for Seq2C.') parser.add_option('--deep-seq', dest='deep_seq', action='store_true', default=False, help='Use run_info_DeepSeq.yaml') parser.add_option('--wgs', dest='is_wgs', action='store_true', default=None, help='Ignore sv_regions and run as WGS') parser.add_option('--only-summary', dest='only_summary', action='store_true', default=False, help='Only generate project-level report') parser.add_option('--jira', dest='jira', help='JIRA case path') parser.add_option('--bed', '--capture', '--amplicons', dest='bed', help='BED file to run targetSeq and Seq2C analysis on.') parser.add_option( '--exons', '--exome', dest='exons', help='Exons BED file to make targetSeq exon/amplicon regions reports.') parser.add_option('--no-prep-bed', dest='prep_bed', help='do not fix input beds and exons', action='store_false', default=True) parser.add_option('--no-dedup', dest='no_dedup', action='store_true', help=SUPPRESS_HELP) parser.add_option('-f', '--freq', '--min-freq', dest='min_freq', type='float', help='Minimum allele frequency for the filtering.') parser.add_option('-o', dest='output_dir', help='Output directory for report combining.') parser.add_option('--transcripts', dest='transcripts_fpath', help='Transcripts for annotation.') parser.add_option('--no-bam2bigwig', dest='no_bam2bigwig', action='store_true', default=False, help=SUPPRESS_HELP) cnf, bcbio_project_dirpaths, bcbio_cnfs, final_dirpaths, tags, is_wgs_in_bcbio, is_rnaseq \ = process_post_bcbio_args(parser) is_wgs = cnf.is_wgs = cnf.is_wgs or is_wgs_in_bcbio cnf.run_date = time.localtime() cnf_project_name = cnf.project_name if len(bcbio_project_dirpaths) > 1: cnf.project_name = None info() info('*' * 70) bcbio_structures = [] for bcbio_project_dirpath, bcbio_cnf, final_dirpath in zip( bcbio_project_dirpaths, bcbio_cnfs, final_dirpaths): bs = BCBioStructure(cnf, bcbio_project_dirpath, bcbio_cnf, final_dirpath, is_wgs=is_wgs, is_rnaseq=is_rnaseq) bcbio_structures.append(bs) # Post-processing one bcbio project as usually if len(bcbio_structures) == 1: if cnf.min_freq is not None: info('Min freq for filtering is %f' % cnf.min_freq) if cnf.steps and cnf.load_mongo and 'MongoLoader' not in cnf.steps: cnf.steps.append('MongoLoader') check_system_resources(cnf, required=['qsub']) bcbio_structure = bcbio_structures[0] bcbio_runner = BCBioRunner(cnf, bcbio_structure, cnf.bcbio_cnf) bcbio_runner.post_jobs() # Special case: multiple projects in input. No post-processing them, but rather combining summary reports together. elif len(bcbio_structures) > 1: if cnf_project_name: cnf.project_name = cnf_project_name else: cnf.project_name = '_'.join( [bs.project_name for bs in bcbio_structures]) if not cnf.output_dir: cnf.output_dir = join(os.getcwd(), cnf.project_name) safe_mkdir(cnf.output_dir) cnf.log_dir = join(cnf.output_dir, 'log') info('log_dirpath: ' + cnf.log_dir) safe_mkdir(cnf.log_dir) set_up_log(cnf, 'miltiple_projects', cnf.project_name, cnf.output_dir) cnf.work_dir = adjust_path(join(cnf.output_dir, 'work')) safe_mkdir(cnf.work_dir) safe_mkdir(adjust_path(join(cnf.output_dir, 'config'))) combine_projects(cnf, bcbio_structures, tags)
def main(args): cnf = read_opts_and_cnfs(extra_opts=[ (['--bam'], dict(dest='bam', help='a path to the BAM file to study')), (['-1'], dict(dest='l_fpath')), (['-2'], dict(dest='r_fpath')), (['--bed', '--capture', '--amplicons'], dict(dest='bed', help='a BED file for capture panel or amplicons')), (['--exons', '--exome', '--features'], dict( dest='features', help= 'a BED file with real CDS/Exon/Gene/Transcript regions with annotations (default "features" is in system_config)' )), (['--exons-no-genes', '--features-no-genes'], dict( dest='features_no_genes', help= 'a BED file with real CDS/Exon regions with annotations, w/o Gene/Transcript records (default "features" is in system_config)' )), (['--original-bed'], dict(dest='original_target_bed', help=SUPPRESS_HELP)), (['--original-exons', '--original-features'], dict( dest='original_features_bed', help='original features genes bed file path (just for reporting)') ), (['--reannotate'], dict(dest='reannotate', help='re-annotate BED file with gene names', action='store_true', default=False)), (['--no-prep-bed'], dict(dest='prep_bed', help='do not fix input beds and exons', action='store_false', default=True)), (['-e', '--extended'], dict(dest='extended', help='extended - flagged regions and missed variants', action='store_true', default=False)), (['--genes'], dict(dest='genes', help='custom list of genes')), (['--padding'], dict( dest='padding', help= 'integer indicating the number of bases to extend each target region up and down-stream. ' 'Default is ' + str(defaults['coverage_reports']['padding']), type='int')), (['--no-dedup'], dict(dest='no_dedup', action='store_true', help=SUPPRESS_HELP)), (['--downsample-to'], dict(dest='downsample_to', type='int', help=SUPPRESS_HELP)), (['--downsampled'], dict(dest='downsampled', action='store_true', help=SUPPRESS_HELP)), (['--fastqc-dirpath'], dict(dest='fastqc_dirpath', help=SUPPRESS_HELP)) ], file_keys=['bam', 'l_fpath', 'r_fpath', 'bed'], key_for_sample_name='bam') if cnf.padding: cnf.coverage_reports.padding = cnf.padding check_system_resources(cnf, required=['bedtools'], optional=[]) check_genome_resources(cnf) features_bed = adjust_path(cnf.features) if cnf.features else adjust_path( cnf.genome.features) if features_bed: info('Features: ' + features_bed) features_bed = verify_file(features_bed) else: info('No features BED found') if cnf.bed: cnf.bed = verify_file(cnf.bed, is_critical=True) info('Using amplicons/capture panel ' + cnf.bed) elif features_bed: info('WGS, taking CDS as target') cnf.bam = verify_bam(cnf.bam, is_critical=True) reports = process_one(cnf, cnf.output_dir, cnf.bam, features_bed=features_bed, features_no_genes_bed=cnf.features_no_genes) summary_report, gene_report = reports[:2] info('') info('*' * 70) if summary_report.txt_fpath: info('Summary report: ' + summary_report.txt_fpath) if gene_report: if gene_report.txt_fpath: info('All regions: ' + gene_report.txt_fpath + ' (' + str(len(gene_report.rows)) + ' regions)') if len(reports) > 2: selected_regions_report = reports[2] if selected_regions_report.txt_fpath: info('Flagged regions: ' + selected_regions_report.txt_fpath + ' (' + str(len(selected_regions_report.rows)) + ' regions)') for fpaths in reports: if fpaths: ok = True info('Checking expected results...') if not isinstance(fpaths, list): fpaths = [fpaths] for fpath in fpaths: if isinstance(fpath, basestring): if not verify_file(fpath): ok = False if ok: info('The results are good.') if not cnf['keep_intermediate']: shutil.rmtree(cnf['work_dir'])
def main(): if len(sys.argv[1]) < 0: critical('Usage: ' + __file__ + ' Input_BED_file -g hg19 -o Annotated_BED_file') input_bed_fpath = verify_bed(sys.argv[1], is_critical=True, description='Input BED file for ' + __file__) cnf = read_opts_and_cnfs( description= 'Annotating BED file based on reference features annotations.', extra_opts=[ (['--reference'], dict(dest='reference')), ], required_keys=['output_file'], file_keys=['reference'], key_for_sample_name=None, fpath_for_sample_name=input_bed_fpath, main_output_is_file=True) check_system_resources(cnf) check_genome_resources(cnf) chr_order = get_chrom_order(cnf) features_fpath = adjust_path(cnf.genome.bed_annotation_features) if not verify_bed(features_fpath, 'Annotated reference BED file'): critical('Annotated reference is required') # features_and_beds = _split_reference_by_priority(cnf, features_fpath) bed = BedTool(input_bed_fpath).cut([0, 1, 2]) info() annotated = None off_targets = None for feature in ['CDS', 'Exon', 'Transcript', 'Gene']: if bed: info('Extracting ' + feature + ' features from ' + features_fpath) features_bed = BedTool(features_fpath).filter( lambda x: x[6] == feature) info('Annotating based on ' + feature) new_annotated, off_targets = _annotate(cnf, bed, features_bed, chr_order) if not annotated: annotated = new_annotated for a in annotated: a.feature = feature else: annotated.extend(new_annotated) if off_targets: bed = BedTool([(r.chrom, r.start, r.end) for r in off_targets]) # off_target_fpath = _save_regions(off_targets, join(work_dirpath, 'off_target_1.bed')) # log('Saved off target1 to ' + str(off_target_fpath)) info() if annotated is not None and off_targets is not None: annotated.extend(off_targets) info() info('Saving annotated regions to ' + str(cnf.output_file)) with open(cnf.output_file, 'w') as out: for region in sorted(annotated, key=lambda r: r.get_key()): out.write(str(region)) # for r, overlap_size in overlaps: # sys.stdout.write('\t' + '\t'.join([ # r.chrom, '{:,}'.format(r.start), '{:,}'.format(r.end), r.gene, r.exon, str(r.strand), r.feature, r.biotype, # str(overlap_size), # '{:.2f}%'.format(100.0 * overlap_size / (r.end - r.start)) # ])) # sys.stdout.write('\n') info('Done.')
def main(args): cnf = read_opts_and_cnfs( extra_opts=[ (['--vcf', '--var'], dict( dest='vcf', help='variants to filter') ), (['--vcf2txt'], dict( dest='vcf2txt', help='variants in vcf2txt to filter') ), (['--cohort-freqs'], dict( dest='cohort_freqs_fpath', help='frequencies of variants in a cohort') ), (['--qc'], dict( dest='qc', action='store_true', default=True, help=SUPPRESS_HELP) ), (['--no-qc'], dict( dest='qc', action='store_false', help=SUPPRESS_HELP) ), (['--no-tsv'], dict( dest='tsv', action='store_false', default=True, help=SUPPRESS_HELP) ), ], required_keys=['vcf'], file_keys=['vcf'], key_for_sample_name='vcf', proc_name=source.varfilter_name + '_post') check_system_resources(cnf, required=['perl']) check_genome_resources(cnf) if not cnf.output_file: cnf.output_file = join(cnf.output_dir, (cnf.caller or 'variants') + '.txt') safe_mkdir(dirname(cnf.output_file)) safe_mkdir(cnf.output_dir) if cnf.vcf.endswith('.vcf.gz') or cnf.vcf.endswith('.vcf'): verify_vcf(cnf.vcf, is_critical=True) if not cnf.vcf2txt: vcf2txt_res_fpath = run_vcf2txt(cnf, {cnf.sample: cnf.vcf}, cnf.output_file) if not vcf2txt_res_fpath: critical('vcf2txt run returned non-0') info('Saved vcf2txt output to ' + vcf2txt_res_fpath) else: cnf.vcf2txt = verify_file(cnf.vcf2txt, is_critical=True) info('Input is vcf2txt output, grepping by sample name ' + cnf.sample) vcf2txt_res_fpath = cnf.output_file with file_transaction(cnf.work_dir, vcf2txt_res_fpath) as tx: with open(cnf.vcf2txt) as f, open(tx, 'w') as out: for i, l in enumerate(f): if l.strip(): if i == 0: out.write(l) else: if l.split('\t')[0] == cnf.sample: out.write(l) info('Using vcf2txt from ' + vcf2txt_res_fpath) # if is_local(): # vardict2mut_pl = get_script_cmdline(cnf, 'perl', join('VarDict', 'vardict2mut.pl')) # info('Running vardict2mut perl') # res = run_vardict2mut(cnf, vcf2txt_res_fpath, # add_suffix(vcf2txt_res_fpath, source.mut_pass_suffix + '_perl'), # vardict2mut_executable=vardict2mut_pl) # if not res: # critical('vardict2mut.pl run returned non-0') mut_fpath = run_vardict2mut(cnf, vcf2txt_res_fpath, add_suffix(vcf2txt_res_fpath, variant_filtering.mut_pass_suffix)) if not mut_fpath: err('vardict2mut failed') else: info('Saved passed mutations to ' + mut_fpath) var_s = source.VarSample(cnf.sample, cnf.output_dir) var_s.anno_vcf_fpath = cnf.vcf var_s.varfilter_dirpath = var_s.dirpath ungz_anno_vcf_fpath = var_s.anno_vcf_fpath if not var_s.anno_vcf_fpath.endswith('.gz') else splitext(var_s.anno_vcf_fpath)[0] ungz_filt_vcf_fpath = join(cnf.output_dir, add_suffix(basename(ungz_anno_vcf_fpath), 'filt')) var_s.filt_vcf_fpath = ungz_filt_vcf_fpath + '.gz' var_s.variants_fpath = vcf2txt_res_fpath var_s.variants_pass_fpath = add_suffix(vcf2txt_res_fpath, source.mut_pass_suffix) ungz_pass_filt_vcf_fpath = add_suffix(ungz_filt_vcf_fpath, 'pass') var_s.pass_filt_vcf_fpath = add_suffix(var_s.filt_vcf_fpath, 'pass') filt_vcf = write_vcf(cnf, var_s, cnf.output_dir, cnf.caller, vcf2txt_res_fpath, mut_fpath) index_vcf(cnf, var_s.name, filt_vcf, cnf.caller) index_vcf(cnf, var_s.name, ungz_pass_filt_vcf_fpath, cnf.caller) if cnf.qc: report = qc.make_report(cnf, var_s.pass_filt_vcf_fpath, var_s) qc_dirpath = join(cnf.output_dir, 'qc') safe_mkdir(qc_dirpath) qc.save_report(cnf, report, var_s, cnf.caller, qc_dirpath, source.varqc_after_name) info('Saved QC to ' + qc_dirpath + ' (' + report.html_fpath + ')') info('-' * 70) info() if not cnf['keep_intermediate']: shutil.rmtree(cnf['work_dir']) info() info('*' * 70) info('Done filtering ' + var_s.name)