def submit_job(cnf, cmdline, job_name, wait_for_steps=None, threads=1, output_fpath=None, stdout_to_outputfile=True, run_on_chara=False, **kwargs): prefix = str(cnf.project_name) + '_' if job_name: prefix += job_name + '_' prefix += datetime.now().strftime("%Y_%m_%d_%H_%M_%S") + '_' f, done_marker_fpath = make_tmpfile(cnf, prefix=prefix, suffix='.done') f, error_marker_fpath = make_tmpfile(cnf, prefix=prefix, suffix='.error') if isfile(done_marker_fpath): os.remove(done_marker_fpath) if isfile(error_marker_fpath): os.remove(error_marker_fpath) job_id = basename(splitext(done_marker_fpath)[0]) tx_output_fpath = None if output_fpath: if cnf.reuse_intermediate and verify_file(output_fpath, silent=True): info(output_fpath + ' exists, reusing') j = JobRunning(None, None, None, None, None, output_fpath=output_fpath, **kwargs) j.is_done = True return j if stdout_to_outputfile: tx_output_fpath = output_fpath + '.tx' if isfile(tx_output_fpath): os.remove(tx_output_fpath) cmdline += ' > ' + tx_output_fpath else: if isfile(output_fpath): os.remove(output_fpath) qsub = get_system_path(cnf, 'qsub', is_critical=True) bash = get_system_path(cnf, 'bash', is_critical=True) if cnf.log_dir: err_fpath = log_fpath = join(cnf.log_dir, job_id + '.log') else: fd, fpath = make_tmpfile(cnf, suffix=job_id + '.log', text=True) err_fpath = log_fpath = fpath queue = cnf.queue runner_script = adjust_system_path(cnf.qsub_runner) verify_file(runner_script, is_critical=True, description='qsub_runner') hold_jid_line = '-hold_jid ' + ','.join(wait_for_steps or ['_']) mem = threads * 15 priority = 0 if cnf.qsub_priority: priority = cnf.qsub_priority extra_qsub_opts = '' if run_on_chara and is_us(): extra_qsub_opts += '-l h="chara|rask"' cmdline = cmdline.replace('"', '\\"').replace('\\\\"', '\\"') qsub_cmdline = ( '{qsub} -pe smp {threads} {extra_qsub_opts} -S {bash} -q {queue} -p {priority} ' '-j n -o {log_fpath} -e {err_fpath} {hold_jid_line} ' '-N {job_id} {runner_script} {done_marker_fpath} {error_marker_fpath} "{cmdline}"'.format(**locals())) info('Submitting job ' + job_id) info(qsub_cmdline) job = JobRunning(job_id, log_fpath, qsub_cmdline, done_marker_fpath, error_marker_fpath, output_fpath=output_fpath, tx_output_fpath=tx_output_fpath, stdout_to_outputfile=stdout_to_outputfile, **kwargs) call(cnf, qsub_cmdline, silent=True) return job
def check_genome_resources(cnf): if cnf.genome is None: critical('Please, specify genome build (one of available in ' + cnf.sys_cnf + ') using the --genome option (e.g., --genome hg38).') if not cnf.genomes: critical('"genomes" section is not specified in system config ' + cnf.sys_cnf) info('Genome: ' + str(cnf.genome.name)) for key in cnf.genome.keys(): if key != 'name' and isinstance(cnf.genome[key], basestring): cnf.genome[key] = adjust_system_path(cnf.genome[key]) if not verify_obj_by_path(cnf.genome[key], key, silent=True): if not cnf.genome[key].endswith('.gz') and verify_file( cnf.genome[key] + '.gz', silent=True): gz_fpath = cnf.genome[key] + '.gz' if verify_file(gz_fpath, silent=True): cnf.genome[key] = gz_fpath if not cnf.genome.features or not cnf.genome.bed_annotation_features or not cnf.genome.cds: warn( 'Warning: features and bed_annotation_features and cds in the system config (' + cnf.sys_cnf + ') must be specified.') if not cnf.transcripts_fpath: cnf.transcripts_fpath = cnf.transcripts_fpath or get_canonical_transcripts( cnf.genome.name, ensembl=True)
def proc_args(argv): info(' '.join(sys.argv)) info() description = 'This script generates target QC reports for each BAM provided as an input. ' \ 'Usage: ' + basename(__file__) + ' sample2bam.tsv --bed target.bed --contols sample1:sample2 -o results_dir' parser = OptionParser(description=description, usage=description) add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser) parser.add_option('-o', dest='output_dir', metavar='DIR', default=join(os.getcwd(), 'seq2c')) parser.add_option('--bed', dest='bed', help='BED file to run Seq2C analysis') parser.add_option('-c', '--controls', dest='controls', help='Optional control sample names for Seq2C. For multiple controls, separate them using :') parser.add_option('--seq2c-opts', dest='seq2c_opts', help='Options for the final lr2gene.pl script.') parser.add_option('--no-prep-bed', dest='prep_bed', help=SUPPRESS_HELP, action='store_false', default=True) (opts, args) = parser.parse_args() logger.is_debug = opts.debug if len(args) == 0: parser.print_usage() sys.exit(1) if len(args) == 1 and not args[0].endswith('.bam'): sample_names, bam_fpaths = read_samples(verify_file(args[0], is_critical=True, description='Input sample2bam.tsv')) bam_by_sample = OrderedDict() for s, b in zip(sample_names, bam_fpaths): bam_by_sample[s] = b else: bam_by_sample = find_bams(args) run_cnf = determine_run_cnf(opts, is_wgs=not opts.__dict__.get('bed')) cnf = Config(opts.__dict__, determine_sys_cnf(opts), run_cnf) check_genome_resources(cnf) cnf.output_dir = adjust_path(cnf.output_dir) verify_dir(dirname(cnf.output_dir), is_critical=True) safe_mkdir(cnf.output_dir) if not cnf.project_name: cnf.project_name = basename(cnf.output_dir) info('Project name: ' + cnf.project_name) cnf.proc_name = 'Seq2C' set_up_dirs(cnf) samples = [ source.TargQC_Sample(name=s_name, dirpath=join(cnf.output_dir, s_name), bam=bam_fpath) for s_name, bam_fpath in bam_by_sample.items()] info('Samples: ') for s in samples: info(' ' + s.name) samples.sort(key=lambda _s: _s.key_to_sort()) target_bed = verify_bed(cnf.bed, is_critical=True) if cnf.bed else None if not cnf.only_summary: cnf.qsub_runner = adjust_system_path(cnf.qsub_runner) if not cnf.qsub_runner: critical('Error: qsub-runner is not provided is sys-config.') verify_file(cnf.qsub_runner, is_critical=True) return cnf, samples, target_bed, cnf.output_dir
def get_system_path(cnf, interpreter_or_name, name=None, extra_warning='', suppress_warn=False, is_critical=False): """ "name" can be: - key in system_into.yaml - relative path in the project (e.g. external/...) - anything in system path """ interpreter = interpreter_or_name if name is None: name = interpreter_or_name interpreter = None if interpreter: if interpreter == 'java': return get_java_tool_cmdline(cnf, name, extra_warning, suppress_warn, is_critical=is_critical) return get_script_cmdline(cnf, interpreter, name, extra_warning=extra_warning, suppress_warn=suppress_warn, is_critical=is_critical) # IN SYSTEM CONFIG? if cnf and (cnf.resources is not None and name.lower() in cnf.resources and 'path' in cnf.resources[name.lower()]): tool_path = cnf.resources[name.lower()]['path'] tool_path = adjust_system_path(tool_path) return verify_obj_by_path(tool_path, name, is_critical=is_critical) # IN PROJECT ROOT DIR? IN EXTERNAL? for dirpath in [code_base_path]: tool_path = join(dirpath, name) if exists(tool_path): return verify_obj_by_path(tool_path, name, is_critical=is_critical) # IN PATH? tool_path = which(name) if tool_path and exists(tool_path): return verify_obj_by_path(tool_path, name, is_critical=is_critical) msg = (name + ' was not found. You may either specify path in the system ' 'config, or load into your PATH environment variable. ' + extra_warning) if not suppress_warn: err(msg) if is_critical: critical(msg) return None
def check_system_resources(cnf, required=list(), optional=list()): to_exit = False for program in required: if not which(program): if cnf.resources is None: critical('No "resources" section in system config.') data = cnf.resources.get(program) if data is None: err(program + ' is required. Specify path in system config or in your environment.' ) to_exit = True else: if 'module' in data: os.system('module load ' + data['module']) # if 'path' not in data: # data['path'] = program elif 'path' in data: data['path'] = adjust_system_path(data['path']) if not isdir(data['path']) and not file_exists( data['path']): err(data['path'] + ' does not exist.') to_exit = True for program in optional: resources = cnf.get('resources') if not resources: break data = resources.get(program) if data is None: continue else: data['path'] = adjust_system_path(data['path']) if not isdir(data['path']) and not file_exists(data['path']): err(data['path'] + ' does not exist.') to_exit = True if to_exit: exit()
def process_all(cnf, bcbio_structure): samples = bcbio_structure.samples key_gene_by_name, use_custom_panel = get_key_or_target_bed_genes( cnf.bed, verify_file(adjust_system_path(cnf.key_genes), 'key genes')) key_or_target_genes = 'target' if use_custom_panel else 'key' mutations = {} for sample in samples: mutations[sample.name] = parse_mutations(cnf, sample, key_gene_by_name, cnf.mutations_fpath, key_or_target_genes, for_flagged_report=True) _generate_summary_flagged_regions_report(cnf, bcbio_structure, samples, mutations, key_or_target_genes) pass
def main(): cnf = get_args() vardict_res_fpath = cnf.mutations_fpath seq2c_tsv_fpath = cnf.seq2c_tsv_fpath sv_fpath = cnf.sv_fpath output_dir = cnf.output_dir sample_name = cnf.sample cytoband = None if 'hg38' in cnf.genome.name: cytoband = cnf.genome.circos_cytoband elif cnf.genome.name == 'hg19': cytoband = 'hg19' if not cytoband: critical('Circos plot does not support ' + cnf.genome + ' genome') svs_bed_fpath = join(output_dir, sample_name + '_svs.bed') parse_svs(cnf, sv_fpath, svs_bed_fpath) if not exists(svs_bed_fpath): return None modified_seq2c_fpath = join(output_dir, sample_name + '_seq2c.tsv') key_genes_chrom, _ = get_key_or_target_bed_genes(cnf.bed_fpath, verify_file(adjust_system_path(cnf.key_genes), 'key genes')) modify_seq2c(cnf, key_genes_chrom, seq2c_tsv_fpath, modified_seq2c_fpath) out_r_script = join(output_dir, sample_name + '.R') with open(out_r_script, 'w') as out_r_script_handle: out_r_script_handle.write(_circos_R_script.format(vardict_all=vardict_res_fpath, seq2c=modified_seq2c_fpath, outdir=output_dir, cytoband=cytoband, mysample=sample_name, svsbed=svs_bed_fpath)) r_script = get_system_path(cnf, 'rscript') cmdline = '{r_script} {out_r_script}'.format(**locals()) res = call(cnf, cmdline)
def generate_flagged_regions_report(cnf, output_dir, sample, ave_depth, gene_by_key): depth_threshs = cnf.coverage_reports.depth_thresholds report = PerRegionSampleReport( sample=sample, metric_storage=get_detailed_metric_storage(depth_threshs)) report.add_record('Sample', sample.name) safe_mkdir(sample.flagged_regions_dirpath) ''' 1. Detect depth threshold (ave sample coverage * DEPTH_THRESH_FROM_AVE_COV) 2. Select regions covered in less than MIN_DEPTH_PERCENT_AT_THRESH at threshold 3. Sort by % at threshold 4. Select those parts of those regions where % = 0, save to BED 5. Find HotSpots at those regions 6. Intersect HotSpots with tracks For each gene where are regions with parts % = 0: sort them by part where % = 0 ''' #vcf_dbs = ['oncomine', 'dbsnp', 'cosmic'] vcf_dbs = ['oncomine'] from source._deprecated_clinical_reporting.clinical_parser import get_key_or_target_bed_genes key_genes, _ = get_key_or_target_bed_genes( cnf.bed, verify_file(adjust_system_path(cnf.key_genes), 'key genes')) depth_cutoff = get_depth_cutoff(ave_depth, depth_threshs) genes_sorted = sorted(gene_by_key.values()) min_cov, max_cov = min_and_max_based_on_outliers(genes_sorted) for coverage_type in ['low', 'high']: info('Selecting and saving ' + coverage_type + ' covered genes') selected_genes = [] if coverage_type == 'low': selected_genes = [ g for g in genes_sorted if g.gene_name in key_genes and (any( e.rates_within_threshs[depth_cutoff] < MIN_DEPTH_PERCENT_AT_THRESH for e in g.get_exons()) or any( a.rates_within_threshs[depth_cutoff] < MIN_DEPTH_PERCENT_AT_THRESH for a in g.get_amplicons())) ] else: if max_cov: selected_genes = [ g for g in genes_sorted if g.gene_name in key_genes and (any( e.avg_depth > max_cov for e in g.get_exons()) or any( a.avg_depth > max_cov for a in g.get_amplicons())) ] for region_type in ['exons', 'target']: selected_regions = [] for gene in selected_genes: if coverage_type == 'low': cur_regions = [ a for a in (gene.get_amplicons() if region_type == 'target' else gene.get_exons()) if a.rates_within_threshs[depth_cutoff] < MIN_DEPTH_PERCENT_AT_THRESH and 'Multi' not in a.feature ] else: cur_regions = [ a for a in (gene.get_amplicons() if region_type == 'target' else gene.get_exons()) if a.avg_depth > max_cov and 'Multi' not in a.feature ] selected_regions.extend(cur_regions) if selected_regions: selected_regions_bed_fpath = join( sample.flagged_regions_dirpath, coverage_type + '_cov_' + region_type + '.bed') save_regions_to_bed(cnf, selected_regions, selected_regions_bed_fpath) # Report cov for Hotspots for db in vcf_dbs: res = _report_normalize_coverage_for_variant_sites( cnf, sample, ave_depth, db, selected_regions_bed_fpath, selected_regions, depth_cutoff, region_type, coverage_type) if not res: return None report = make_flat_region_report(sample, selected_regions, depth_threshs) flagged_txt_fpath = add_suffix( add_suffix(sample.flagged_txt, region_type), coverage_type) flagged_tsv_fpath = add_suffix( add_suffix(sample.flagged_tsv, region_type), coverage_type) report.save_txt(flagged_txt_fpath) report.save_tsv(flagged_tsv_fpath) info('') info(coverage_type + ' covered ' + region_type + '(total ' + str(len(selected_regions)) + ') for sample ' + sample.name + ' saved into:') info(' ' + flagged_txt_fpath + ', ' + flagged_tsv_fpath) return report
def main(): info(' '.join(sys.argv)) info() description = 'This script generates target QC reports for each BAM provided as an input.' parser = OptionParser(description=description) add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser, threads=1) parser.add_option('--work-dir', dest='work_dir', metavar='DIR') parser.add_option('--log-dir', dest='log_dir') parser.add_option('--only-summary', dest='only_summary', action='store_true') parser.add_option('-o', dest='output_dir', metavar='DIR', default=join(os.getcwd(), 'targetqc')) parser.add_option('--reannotate', dest='reannotate', action='store_true', default=False, help='re-annotate BED file with gene names') parser.add_option('--dedup', dest='dedup', action='store_true', default=False, help='count duplicates in coverage metrics') parser.add_option('--bed', dest='bed', help='BED file to run targetSeq and Seq2C analysis on.') parser.add_option( '--exons', '--exome', '--features', dest='features', help= 'Annotated CDS/Exon/Gene/Transcripts BED file to make targetSeq exon/amplicon regions reports.' ) (opts, args) = parser.parse_args() logger.is_debug = opts.debug if len(args) == 0: critical('No BAMs provided to input.') bam_fpaths = list(set([abspath(a) for a in args])) bad_bam_fpaths = [] for fpath in bam_fpaths: if not verify_bam(fpath): bad_bam_fpaths.append(fpath) if bad_bam_fpaths: critical('BAM files cannot be found, empty or not BAMs:' + ', '.join(bad_bam_fpaths)) run_cnf = determine_run_cnf(opts, is_wgs=not opts.__dict__.get('bed')) cnf = Config(opts.__dict__, determine_sys_cnf(opts), run_cnf) if not cnf.project_name: cnf.project_name = basename(cnf.output_dir) info('Project name: ' + cnf.project_name) cnf.proc_name = 'TargQC' set_up_dirs(cnf) # cnf.name = 'TargQC_' + cnf.project_name check_genome_resources(cnf) verify_bed(cnf.bed, is_critical=True) bed_fpath = adjust_path(cnf.bed) info('Using amplicons/capture panel ' + bed_fpath) features_bed_fpath = adjust_path( cnf.features) if cnf.features else adjust_path(cnf.genome.features) info('Features: ' + features_bed_fpath) genes_fpath = None if cnf.genes: genes_fpath = adjust_path(cnf.genes) info('Custom genes list: ' + genes_fpath) if not cnf.only_summary: cnf.qsub_runner = adjust_system_path(cnf.qsub_runner) if not cnf.qsub_runner: critical('Error: qsub-runner is not provided is sys-config.') verify_file(cnf.qsub_runner, is_critical=True) info('*' * 70) info() targqc_html_fpath = run_targqc(cnf, cnf.output_dir, bam_fpaths, bed_fpath, features_bed_fpath, genes_fpath) if targqc_html_fpath: send_email( cnf, 'TargQC report for ' + cnf.project_name + ':\n ' + targqc_html_fpath)
def proc_args(argv): info(' '.join(sys.argv)) info() description = 'This script generates target QC reports for each BAM provided as an input.' parser = OptionParser(description=description) add_cnf_t_reuse_prjname_donemarker_workdir_genome_debug(parser) parser.add_option('--log-dir', dest='log_dir') parser.add_option('--is-wgs', dest='is_wgs', action='store_true', default=False, help='whole genome sequencing') parser.add_option('--is-deep-seq', dest='is_deep_seq', action='store_true', default=False, help='deep targeted sequencing') parser.add_option('--only-summary', dest='only_summary', action='store_true') parser.add_option('-o', dest='output_dir', metavar='DIR', default=join(os.getcwd(), 'targetqc')) parser.add_option('-c', '--caller', dest='caller') parser.add_option('--qc', dest='qc', action='store_true', default=False) parser.add_option('--no-qc', dest='qc', action='store_false', default=False) parser.add_option('--qc-caption', dest='qc_caption', help=SUPPRESS_HELP) parser.add_option('--no-tsv', dest='tsv', action='store_false', default=True, help=SUPPRESS_HELP) (opts, args) = parser.parse_args() logger.is_debug = opts.debug if len(args) == 0: critical('No vcf files provided to input.') run_cnf = determine_run_cnf(opts, is_targetseq=opts.is_deep_seq, is_wgs=opts.is_wgs) cnf = Config(opts.__dict__, determine_sys_cnf(opts), run_cnf) vcf_fpath_by_sample = read_samples(args, cnf.caller) info() if not cnf.project_name: cnf.project_name = basename(cnf.output_dir) info('Project name: ' + cnf.project_name) cnf.proc_name = 'Variants' set_up_dirs(cnf) # cnf.name = 'TargQC_' + cnf.project_name info(' '.join(sys.argv)) samples = [ source.VarSample(s_name, join(cnf.output_dir, s_name), vcf=vcf_fpath) for s_name, vcf_fpath in vcf_fpath_by_sample.items() ] samples.sort(key=lambda _s: _s.key_to_sort()) check_genome_resources(cnf) if not cnf.only_summary: cnf.qsub_runner = adjust_system_path(cnf.qsub_runner) if not cnf.qsub_runner: critical('Error: qsub-runner is not provided is sys-config.') verify_file(cnf.qsub_runner, is_critical=True) return cnf, samples
def _snpeff(cnf, input_fpath): if 'snpeff' not in cnf.annotation or 'snpeff' not in cnf.genome: return None, None, None step_greetings('SnpEff') output_fpath = intermediate_fname(cnf, input_fpath, 'snpEff') stats_fpath = join( cnf.work_dir, cnf.sample + (('-' + cnf.caller) if cnf.caller else '') + '.snpEff_summary.csv') if output_fpath.endswith('.gz'): output_fpath = output_fpath[:-3] if cnf.reuse_intermediate and verify_vcf(output_fpath): info('VCF ' + output_fpath + ' exists, reusing...') return output_fpath, stats_fpath, splitext( stats_fpath)[0] + '.genes.txt' snpeff = get_java_tool_cmdline(cnf, 'snpeff') ref_name = cnf.genome.snpeff.reference or cnf.genome.name if ref_name.startswith('hg19') or ref_name.startswith('GRCh37'): ref_name = 'GRCh37.75' if ref_name.startswith('hg38'): ref_name = 'GRCh38.82' opts = '' if cnf.annotation.snpeff.cancer: opts += ' -cancer' assert cnf.transcripts_fpath, 'Transcript for annotation must be specified!' verify_file(cnf.transcripts_fpath, 'Transcripts for snpEff -onlyTr', is_critical=True) opts += ' -onlyTr ' + cnf.transcripts_fpath + ' ' db_path = adjust_system_path(cnf.genome.snpeff.data) if db_path: opts += ' -dataDir ' + db_path elif cnf.resources.snpeff.config: conf = get_system_path(cnf, cnf.resources.snpeff.config) if conf: opts += ' -c ' + conf + ' ' else: err('Cannot find snpEff config file ' + str(cnf.resources.snpeff.config)) if cnf.annotation.snpeff.extra_options: opts += '' if not cnf.no_check: info('Removing previous snpEff annotations...') res = remove_prev_eff_annotation(cnf, input_fpath) if not res: err('Could not remove preivous snpEff annotations') return None, None, None input_fpath = res snpeff_type = get_snpeff_type(snpeff) if snpeff_type == "old": opts += ' -stats ' + stats_fpath + ' -csvStats' else: opts += ' -csvStats ' + stats_fpath cmdline = '{snpeff} eff {opts} -noLog -i vcf -o vcf {ref_name} {input_fpath}'.format( **locals()) for i in range(1, 20): try: res = call_subprocess(cnf, cmdline, input_fpath, output_fpath, exit_on_error=False, stdout_to_outputfile=True, overwrite=True) except OSError: import traceback, time err(traceback.format_exc()) warn() info('Waiting 1 minute') time.sleep(60) info('Rerunning ' + str(i)) else: break output_fpath = verify_vcf(output_fpath, is_critical=True) snpeff_summary_html_fpath = 'snpEff_summary.html' if isfile(snpeff_summary_html_fpath): info('SnpEff created ' + snpeff_summary_html_fpath + ' in the cwd, removing it...') try: os.remove(snpeff_summary_html_fpath) except OSError: pass if res: return output_fpath, stats_fpath, splitext( stats_fpath)[0] + '.genes.txt' else: return None, None, None