def bgzip_and_tabix(fpath, reuse=False, tabix_parameters='', **kwargs): gzipped_fpath = join(fpath + '.gz') tbi_fpath = gzipped_fpath + '.tbi' if reuse and \ file_exists(gzipped_fpath) and (getctime(gzipped_fpath) >= getctime(fpath) if file_exists(fpath) else True) and \ file_exists(tbi_fpath) and getctime(tbi_fpath) >= getctime(gzipped_fpath): info('Actual compressed file and index exist, reusing') return gzipped_fpath info('Compressing and tabixing file, writing ' + gzipped_fpath + '(.tbi)') bgzip = which('bgzip') tabix = which('tabix') if not bgzip: err('Cannot index file because bgzip is not found') if not tabix: err('Cannot index file because tabix is not found') if not bgzip and not tabix: return fpath if isfile(gzipped_fpath): os.remove(gzipped_fpath) if isfile(tbi_fpath): os.remove(tbi_fpath) info('BGzipping ' + fpath) cmdline = '{bgzip} {fpath}'.format(**locals()) call_process.run(cmdline) info('Tabixing ' + gzipped_fpath) cmdline = '{tabix} {tabix_parameters} {gzipped_fpath}'.format(**locals()) call_process.run(cmdline) return gzipped_fpath
def bgzip_and_tabix(fpath, reuse=False, tabix_parameters='', **kwargs): gzipped_fpath = join(fpath + '.gz') tbi_fpath = gzipped_fpath + '.tbi' if reuse and \ file_exists(gzipped_fpath) and (getctime(gzipped_fpath) >= getctime(fpath) if file_exists(fpath) else True) and \ file_exists(tbi_fpath) and getctime(tbi_fpath) >= getctime(gzipped_fpath): info('Actual compressed file and index exist, reusing') return gzipped_fpath info('Compressing and tabixing file, writing ' + gzipped_fpath + '(.tbi)') bgzip = which('bgzip') tabix = which('tabix') if not bgzip: err('Cannot index file because bgzip is not found') if not tabix: err('Cannot index file because tabix is not found') if not bgzip and not tabix: return fpath if isfile(gzipped_fpath): os.remove(gzipped_fpath) if isfile(tbi_fpath): os.remove(tbi_fpath) info('BGzipping ' + fpath) cmdline = '{bgzip} {fpath}'.format(**locals()) call_process.run(cmdline) info('Tabixing ' + gzipped_fpath) cmdline = '{tabix} {tabix_parameters} {gzipped_fpath}'.format(**locals()) call_process.run(cmdline) return gzipped_fpath
def sort_bed_gsort(input_bed_fpath, output_bed_fpath=None, work_dir=None, fai_fpath=None, genome=None): input_bed_fpath = verify_bed(input_bed_fpath, is_critical=True) output_bed_fpath = adjust_path(output_bed_fpath) if output_bed_fpath \ else intermediate_fname(work_dir, input_bed_fpath, 'sorted') debug('Sorting regions in ' + str(input_bed_fpath)) if can_reuse(output_bed_fpath, input_bed_fpath): debug(output_bed_fpath + ' exists, reusing') return output_bed_fpath if fai_fpath: fai_fpath = verify_file(fai_fpath) elif genome: fai_fpath = verify_file(ref.get_fai(genome)) else: critical('Either of fai_fpath or genome build name must be specified') with file_transaction(work_dir, output_bed_fpath) as tx: run('gsort {input_bed_fpath} {fai_fpath}'.format(**locals()), output_fpath=tx) return output_bed_fpath
def build_snps_panel(bcbio_projs=None, bed_files=None, output_dir=None, genome=None): selected_snps_file = join(output_dir, 'snps.bed') if can_reuse(selected_snps_file, bed_files): return selected_snps_file work_dir = safe_mkdir(join(output_dir, 'work')) log.info('Intersecting BED files for projects.') all_bed_files = set() for proj in bcbio_projs or []: if proj.coverage_bed: log.info(proj.project_name + ': selecting ' + proj.coverage_bed) all_bed_files.add(proj.coverage_bed) else: all_bed_files.add(proj.call) all_bed_files |= set(bed_files or []) overlapped_bed = join(work_dir, 'merged_bed_files.bed') log.info(f'BED files: {all_bed_files}, mergin, writing {overlapped_bed}') overlap_bed_files(all_bed_files, overlapped_bed) # Selecting SNPs from dbSNP dbsnp_file = get_dbsnp(genome) dbsnp_snps_file = join(work_dir, 'snps_in_merged_bed_files.bed') if not can_reuse(dbsnp_snps_file, [dbsnp_file, overlapped_bed]): cmdl = f'bedtools intersect -header -a {dbsnp_file} -b {overlapped_bed}' call_process.run(cmdl, dbsnp_snps_file) subset_bed_file = add_suffix(dbsnp_snps_file, 'subset') _make_snp_file(dbsnp_snps_file, genome, subset_bed_file) shutil.copyfile(subset_bed_file, selected_snps_file) return selected_snps_file
def cut(fpath, col_num, output_fpath=None): output_fpath = output_fpath or add_suffix(fpath, 'cut') if can_reuse(output_fpath, fpath): return output_fpath cmdline = 'cut -f' + ','.join(map(str, range(1, col_num + 1))) + ' ' + fpath + ' > ' + output_fpath call_process.run(cmdline, output_fpath=output_fpath) return output_fpath
def lift_over(fpath, from_genome, to_genome): chain_file = join(dirname(__file__), 'over.chain', f'{from_genome}To{to_genome.title()}.over.chain.gz') if not verify_file(chain_file): log.critical(f'Error: conversion from {from_genome} to {to_genome} is not supported!') out_fpath = add_suffix(fpath, to_genome) call_process.run(f'liftOver {fpath} {chain_file} {out_fpath} {out_fpath}.unMapped') return out_fpath
def get_padded_bed_file(work_dir, bed, padding, fai_fpath): genome_fpath = fai_fpath info('Making bed file for padded regions...') bedtools = which('bedtools') cmdline = '{bedtools} slop -i {bed} -g {genome_fpath} -b {padding}'.format(**locals()) output_fpath = intermediate_fname(work_dir, bed, 'padded') call_process.run(cmdline, output_fpath=output_fpath) return output_fpath
def cut(fpath, col_num, output_fpath=None): output_fpath = output_fpath or add_suffix(fpath, 'cut') if can_reuse(output_fpath, fpath): return output_fpath cmdline = 'cut -f' + ','.join(map(str, range( 1, col_num + 1))) + ' ' + fpath + ' > ' + output_fpath call_process.run(cmdline, output_fpath=output_fpath) return output_fpath
def get_padded_bed_file(work_dir, bed, padding, fai_fpath): genome_fpath = fai_fpath info('Making bed file for padded regions...') bedtools = which('bedtools') cmdline = '{bedtools} slop -i {bed} -g {genome_fpath} -b {padding}'.format( **locals()) output_fpath = intermediate_fname(work_dir, bed, 'padded') call_process.run(cmdline, output_fpath=output_fpath) return output_fpath
def overlap_bed_files(bed_files, output_bed_file): if can_reuse(output_bed_file, bed_files): return output_bed_file if len(bed_files) == 1: shutil.copy(bed_files.pop(), output_bed_file) return output_bed_file cmdl = 'bedops --intersect' + ''.join([' <(sort-bed ' + bf + ')' for bf in bed_files]) call_process.run(cmdl, output_bed_file) return output_bed_file
def intersect_bed(work_dir, bed1, bed2): bed1_fname, _ = splitext_plus(basename(bed1)) bed2_fname, _ = splitext_plus(basename(bed2)) output_fpath = join(work_dir, bed1_fname + '__' + bed2_fname + '.bed') if can_reuse(output_fpath, [bed1, bed2]): return output_fpath bedtools = which('bedtools') cmdline = '{bedtools} intersect -u -a {bed1} -b {bed2}'.format(**locals()) call_process.run(cmdline, output_fpath=output_fpath, checks=[call_process.file_exists_check]) return output_fpath
def call_sambamba(cmdl, bam_fpath, output_fpath=None, command_name='', no_index=False): if not no_index: index_bam(bam_fpath) sambamba = get_executable() run(sambamba + ' ' + cmdl, output_fpath=output_fpath) return output_fpath
def bam_to_bed(bam_fpath, to_gzip=True): debug('Converting the BAM to BED to save some memory.') # from here: http://davetang.org/muse/2015/08/05/creating-a-coverage-plot-using-bedtools-and-r/ bam_bed_fpath = splitext_plus(bam_fpath)[0] + ('.bed.gz' if to_gzip else '.bed') if can_reuse(bam_bed_fpath, bam_fpath): return bam_bed_fpath bedtools = which('bedtools') gzip = which('gzip') cmdline = '{bedtools} bamtobed -i {bam_fpath}'.format(**locals()) cmdline += ' | {gzip}'.format(**locals()) if to_gzip else '' call_process.run(cmdline, output_fpath=bam_bed_fpath) return bam_bed_fpath
def intersect_bed(work_dir, bed1, bed2): bed1_fname, _ = splitext_plus(basename(bed1)) bed2_fname, _ = splitext_plus(basename(bed2)) output_fpath = join(work_dir, bed1_fname + '__' + bed2_fname + '.bed') if can_reuse(output_fpath, [bed1, bed2]): return output_fpath bedtools = which('bedtools') cmdline = '{bedtools} intersect -u -a {bed1} -b {bed2}'.format(**locals()) call_process.run(cmdline, output_fpath=output_fpath, checks=[call_process.file_exists_check]) return output_fpath
def bam_to_bed(bam_fpath, to_gzip=True): debug( 'Converting the BAM to BED to save some memory.' ) # from here: http://davetang.org/muse/2015/08/05/creating-a-coverage-plot-using-bedtools-and-r/ bam_bed_fpath = splitext_plus(bam_fpath)[0] + ('.bed.gz' if to_gzip else '.bed') if can_reuse(bam_bed_fpath, bam_fpath): return bam_bed_fpath bedtools = which('bedtools') gzip = which('gzip') cmdline = '{bedtools} bamtobed -i {bam_fpath}'.format(**locals()) cmdline += ' | {gzip}'.format(**locals()) if to_gzip else '' call_process.run(cmdline, output_fpath=bam_bed_fpath) return bam_bed_fpath
def annotate_target(work_dir, target_bed, genome_build): output_fpath = intermediate_fname(work_dir, target_bed, 'ann') if can_reuse(output_fpath, target_bed): info(output_fpath + ' exists, reusing') return output_fpath bed_annotation = which('annotate_bed.py') if not bed_annotation: bed_annotation = which('bed_annotation') critical('Error: bed_annotation not found in PATH, please install `conda install -c vladsaveliev bed_annotation`.') cmdline = '{bed_annotation} {target_bed} -g {genome_build} -o {output_fpath}'.format(**locals()) run(cmdline, output_fpath, stdout_to_outputfile=False) output_fpath = clean_bed(output_fpath, work_dir) return output_fpath
def _slice_vcf_fn(work_dir, label, vcf_file, overlapped_bed): sliced_vcf_file = join(work_dir, label + '.sliced.vcf') if not can_reuse(sliced_vcf_file, [vcf_file]): run(f'bcftools view {vcf_file} --targets-file {overlapped_bed} -o {sliced_vcf_file}' ) # ann_vcf_file = join(work_dir, label + '.sliced.ann.vcf') # if not can_reuse(ann_vcf_file, [sliced_vcf_file]): # vcf_header = join(work_dir, label + '.vcf_header') # with open(vcf_header, 'w') as f: # f.write('##INFO=<ID=CHROM,Number=1,Type=String,Description="Region chromosome">\n') # f.write('##INFO=<ID=FROM,Number=1,Type=String,Description="Region start">\n') # f.write('##INFO=<ID=TO,Number=1,Type=String,Description="Region end">\n') # run(f'bcftools annotate -c CHROM,FROM,TO -a {overlapped_bed} {sliced_vcf_file} ' # f'-h {vcf_header} -o {ann_vcf_file}') return label, sliced_vcf_file
def annotate_target(work_dir, target_bed, genome_build): output_fpath = intermediate_fname(work_dir, target_bed, 'ann') if can_reuse(output_fpath, target_bed): info(output_fpath + ' exists, reusing') return output_fpath bed_annotation = which('annotate_bed.py') if not bed_annotation: bed_annotation = which('bed_annotation') critical( 'Error: bed_annotation not found in PATH, please install `conda install -c vladsaveliev bed_annotation`.' ) cmdline = '{bed_annotation} {target_bed} -g {genome_build} -o {output_fpath}'.format( **locals()) run(cmdline, output_fpath, stdout_to_outputfile=False) output_fpath = clean_bed(output_fpath, work_dir) return output_fpath
def index_bam(bam_fpath, sambamba=None, samtools=None): sambamba = sambamba or get_executable() indexed_bam = bam_fpath + '.bai' if not can_reuse(indexed_bam, cmp_f=bam_fpath, silent=True): cmdline = '{sambamba} index {bam_fpath}'.format(**locals()) res = run(cmdline, output_fpath=indexed_bam, stdout_to_outputfile=False, stdout_tx=False)
def ungzip_if_needed(cnf, fpath, silent=False): if fpath.endswith('.gz'): fpath = fpath[:-3] if not file_exists(fpath) and file_exists(fpath + '.gz'): gz_fpath = fpath + '.gz' cmdline = 'gunzip -c {gz_fpath}'.format(**locals()) res = run(cmdline, output_fpath=fpath) if not silent: info() if not res: return None return fpath
def sort_bed_gsort(input_bed_fpath, output_bed_fpath=None, work_dir=None, fai_fpath=None, genome=None): input_bed_fpath = verify_bed(input_bed_fpath, is_critical=True) output_bed_fpath = adjust_path(output_bed_fpath) if output_bed_fpath \ else intermediate_fname(work_dir, input_bed_fpath, 'sorted') debug('Sorting regions in ' + str(input_bed_fpath)) if can_reuse(output_bed_fpath, input_bed_fpath): debug(output_bed_fpath + ' exists, reusing') return output_bed_fpath if fai_fpath: fai_fpath = verify_file(fai_fpath) elif genome: fai_fpath = verify_file(ref.get_fai(genome)) else: critical('Either of fai_fpath or genome build name must be specified') with file_transaction(work_dir, output_bed_fpath) as tx: run('gsort {input_bed_fpath} {fai_fpath}'.format(**locals()), output_fpath=tx) return output_bed_fpath
def _calculate(bam_file, work_dir, genome_fasta_file, min_depth): """Calculate coverage in parallel using samtools depth through goleft. samtools depth removes duplicates and secondary reads from the counts: if ( b->core.flag & (BAM_FUNMAP | BAM_FSECONDARY | BAM_FQCFAIL | BAM_FDUP) ) continue; """ output_prefix = os.path.join(work_dir, bam_samplename(bam_file)) callability_annotation_file = output_prefix + '.callable.bed' if not can_reuse(callability_annotation_file, bam_file): info(f'Calculating coverage at {bam_file}') run(f'goleft depth --q 1 --mincov {min_depth} --reference {genome_fasta_file} --ordered' f' --prefix {output_prefix} {bam_file}') callable_file = output_prefix + '.callable.CALLABLE.bed' if not can_reuse(callable_file, callability_annotation_file): with file_transaction(None, callable_file) as tx: pybedtools.BedTool(callability_annotation_file)\ .filter(lambda x: x.name == 'CALLABLE')\ .saveas(tx) return callable_file
def build_tree(run): info('Writing fasta to ' + run.fasta_file_path()) samples = [s for p in run.projects for s in p.samples] with open(run.fasta_file_path(), 'w') as fhw: for s in samples: snps_by_rsid = s.snps_from_run(run) fhw.write('>' + s.long_name() + '\n') fhw.write(''.join(snps_by_rsid[loc.rsid].get_gt() for loc in run.locations.all()) + '\n') info('All fasta saved to ' + run.fasta_file_path()) info() info('Building phylogeny tree using prank...') prank_out = join(run.work_dir_path(), splitext(basename(run.fasta_file_path()))[0]) call_process.run('prank -d=' + run.fasta_file_path() + ' -o=' + prank_out + ' -showtree') if not verify_file(prank_out + '.best.dnd'): critical('Prank failed to run') os.rename(prank_out + '.best.dnd', run.tree_file_path()) os.remove(prank_out + '.best.fas') return run.fasta_file_path()
def index_bam(bam_fpath, sambamba=None, samtools=None): sambamba = sambamba or get_executable() indexed_bam = bam_fpath + '.bai' if not can_reuse(indexed_bam, cmp_f=bam_fpath, silent=True): cmdline = '{sambamba} index {bam_fpath}'.format(**locals()) res = run(cmdline, output_fpath=indexed_bam, stdout_to_outputfile=False, stdout_tx=False)
def call_sambamba(cmdl, bam_fpath, output_fpath=None, command_name='', no_index=False): if not no_index: index_bam(bam_fpath) sambamba = get_executable() run(sambamba + ' ' + cmdl, output_fpath=output_fpath) return output_fpath
def _vardict_pileup_sample(sample, work_dir, output_dir, genome_fasta_file, snp_file): vardict_snp_vars = join(work_dir, sample.name + '_vars.txt') vcf_file = join(output_dir, sample.name + '.vcf') if can_reuse(vardict_snp_vars, [sample.bam, snp_file]) and can_reuse( vcf_file, vardict_snp_vars): return vcf_file vardict_exec = which('vardict') if not vardict_exec: critical( 'Error: vardict is not in PATH. Please install it with `conda install -c bioconda vardict`' ) vardict_bin_dir = dirname(vardict_exec) # Run VarDict index_bam(sample.bam) cmdl = '{vardict_exec} -G {genome_fasta_file} -N {sample.name} -b {sample.bam} -p -D {snp_file}'.format( **locals()) call_process.run(cmdl, output_fpath=vardict_snp_vars) # Complex variants might have a shifted start positions with respect to rsid so we are # associating starts with rsid for futher snp identification ann_by_var = defaultdict(list) with open(vardict_snp_vars) as f: for l in f: fs = l.split('\t') ann, chrom, start = fs[1], fs[2], fs[3] ann_by_var[(chrom, start)] = ann info() info('Converting to VCF') work_vcf_file = join(work_dir, sample.name + '_vars.vcf') cmdl = ( 'cut -f-34 ' + vardict_snp_vars + ' | awk -F"\\t" -v OFS="\\t" \'{for (i=1;i<=NF;i++) { if ($i=="") $i="0" } print $0 }\'' ' | ' + join('teststrandbias.R') + ' | ' + join('var2vcf_valid.pl') + ' -A -f 0.2' + '') call_process.run(cmdl, output_fpath=work_vcf_file) # Fix non-call records with empty REF and LAT, and "NA" values assigned to INFO's SN and HICOV fixed_vcf_file = add_suffix(work_vcf_file, 'fixed') info('Fixing VCF for parsing, writing to ' + fixed_vcf_file) with open(work_vcf_file) as inp, open(fixed_vcf_file, 'w') as out_f: for l in inp: if l.startswith('#'): out_f.write(l) else: fs = l.split('\t') chrom, pos, _, ref, alt = fs[0], int( fs[1]), fs[2], fs[3], fs[4] if alt in ['.', '']: fs[4] = fs[3] = _get_fasta_ref( genome_fasta_file, chrom, pos) # Reading the reference allele from fasta l = '\t'.join(fs) l = l.replace('=NA;', '=.;') l = l.replace('=;', '=.;') l = l.replace('TYPE=0', 'TYPE=REF') out_f.write(l) assert verify_file(fixed_vcf_file) info('Annotating VCF with gene names and rsIDs') ann_vcf_file = add_suffix(fixed_vcf_file, 'ann') with open(fixed_vcf_file) as f, open(ann_vcf_file, 'w') as out: vcf_reader = vcf.Reader(f) vcf_writer = vcf.Writer(out, vcf_reader) for rec in vcf_reader: ann = ann_by_var[(rec.CHROM, str(rec.POS))] rec.ID = ann.split('|')[0] rec.INFO['ANNOTATION'] = ann vcf_writer.write_record(rec) assert verify_file(ann_vcf_file), ann_vcf_file ann_hdr_vcf_file = add_suffix(ann_vcf_file, 'hdr') cmdl = 'bcftools annotate -h <(echo ' \ '\'##INFO=<ID=ANNOTATION,Number=1,Type=String,Description="rsid|gene_name|ref|alts">\') ' + \ bgzip_and_tabix(ann_vcf_file) call_process.run(cmdl, output_fpath=ann_hdr_vcf_file) debug('Renaming ' + ann_hdr_vcf_file + ' -> ' + vcf_file) os.rename(ann_hdr_vcf_file, vcf_file) return vcf_file