def align(work_dir,
sample_name,
l_fpath,
r_fpath,
bwa,
smb,
bwa_prefix,
dedup=True,
threads=1):
info('Running bwa to align reads...')
bam_fpath = make_bam_fpath(work_dir)
if can_reuse(bam_fpath, [l_fpath, r_fpath]):
return bam_fpath
tmp_dirpath = join(work_dir, 'sambamba_tmp_dir')
safe_mkdir(tmp_dirpath)
bwa_cmdline = (
'{bwa} mem -t {threads} -v 2 {bwa_prefix} {l_fpath} {r_fpath} | ' +
'{smb} view /dev/stdin -t {threads} -f bam -S -o - | ' +
'{smb} sort /dev/stdin -t {threads} --tmpdir {tmp_dirpath} -o {bam_fpath}'
).format(**locals())
run(bwa_cmdline, output_fpath=bam_fpath, stdout_to_outputfile=False)
if dedup:
dedup_bam_fpath = add_suffix(bam_fpath, 'dedup')
dedup_cmdl = '{smb} markdup -t {threads} {bam_fpath} {dedup_bam_fpath}'.format(
**locals())
run(dedup_cmdl,
output_fpath=dedup_bam_fpath,
stdout_to_outputfile=False)
verify_bam(dedup_bam_fpath)
os.rename(dedup_bam_fpath, bam_fpath)
sambamba.index_bam(bam_fpath)
# samtools view -b -S -u - |
# sambamba sort -N -t 8 -m 682M --tmpdir /Molly/saveliev/cancer-dream-syn3/work/align/syn3-normal/split/tx/tmpwdXndE/syn3-normal-sort-1_20000000-sorttmp-full
# -o /Molly/saveliev/cancer-dream-syn3/work/align/syn3-normal/split/tx/tmpwdXndE/syn3-normal-sort-1_20000000.bam
# /dev/stdin
# if dedup:
# info()
# info('Calling SamBlaster to mark duplicates')
# markdup_sam_fpath = markdup_sam(sam_fpath, samblaster)
# if markdup_sam_fpath:
# sam_fpath = markdup_sam_fpath
# info()
# info('Converting to BAM')
# cmdline = sambamba.get_executable() + ' view -t {threads} -S -f bam {sam_fpath}'.format(**locals())
# run(cmdline, output_fpath=bam_fpath, reuse=cfg.reuse_intermediate)
#
# info()
# info('Sorting BAM')
# prefix = splitext(sorted_bam_fpath)[0]
# cmdline = sambamba.get_executable() + ' sort -t {threads} {bam_fpath} -o {sorted_bam_fpath}'.format(**locals())
# run(cmdline, output_fpath=sorted_bam_fpath, stdout_to_outputfile=False, reuse=cfg.reuse_intermediate)
return bam_fpath
def run_qualimap(work_dir,
output_dir,
output_fpaths,
bam_fpath,
genome,
bed_fpath=None,
threads=1):
info('Analysing ' + bam_fpath)
safe_mkdir(dirname(output_dir))
safe_mkdir(output_dir)
mem_cmdl = ''
mem_m = get_qualimap_max_mem(bam_fpath)
mem = str(int(mem_m)) + 'M'
mem_cmdl = '--java-mem-size=' + mem
cmdline = (find_executable() +
' bamqc --skip-duplicated -nt {threads} {mem_cmdl} -nr 5000 '
'-bam {bam_fpath} -outdir {output_dir}')
if genome.startswith('hg') or genome.startswith('GRCh'):
cmdline += ' -gd HUMAN'
if genome.startswith('mm'):
cmdline += ' -gd MOUSE'
if bed_fpath:
cmdline += ' -gff {bed_fpath}'
debug('Using amplicons/capture panel ' + bed_fpath)
cmdline = cmdline.format(**locals())
if not all(
can_reuse(fp, [bam_fpath, bed_fpath] if bed_fpath else [bam_fpath])
for fp in output_fpaths):
for fp in output_fpaths:
if isfile(fp):
os.remove(fp)
try:
run(cmdline, env_vars=dict(DISPLAY=None))
except subprocess.CalledProcessError as e:
if 'The alignment file is unsorted.' in e.output:
info()
info('BAM file is unsorted; trying to sort and rerun QualiMap')
sorted_bam_fpath = sort_bam(bam_fpath)
cmdline = cmdline.replace(bam_fpath, sorted_bam_fpath)
run(cmdline, env_vars=dict(DISPLAY=None))
if not all(
verify_file(
fp, cmp_f=[bam_fpath, bed_fpath] if bed_fpath else [bam_fpath])
for fp in output_fpaths):
critical('Some of the QualiMap results were not generated')
return output_dir
def set_up_dirs(proc_name, output_dir=None, work_dir=None, log_dir=None):
""" Creates output_dir, work_dir, and sets up log
"""
output_dir = safe_mkdir(
adjust_path(output_dir or join(os.getcwd(), proc_name)), 'output_dir')
debug('Saving results into ' + output_dir)
work_dir = safe_mkdir(work_dir or join(output_dir, 'work'),
'working directory')
info('Using work directory ' + work_dir)
log_fpath = set_up_log(log_dir or safe_mkdir(join(work_dir, 'log')),
proc_name + '.log')
return output_dir, work_dir, log_fpath
def align(work_dir, sample_name, l_fpath, r_fpath, bwa, smb, bwa_prefix, dedup=True, threads=1):
info('Running bwa to align reads...')
bam_fpath = make_bam_fpath(work_dir)
if can_reuse(bam_fpath, [l_fpath, r_fpath]):
return bam_fpath
tmp_dirpath = join(work_dir, 'sambamba_tmp_dir')
safe_mkdir(tmp_dirpath)
bwa_cmdline = ('{bwa} mem -t {threads} -v 2 {bwa_prefix} {l_fpath} {r_fpath} | ' +
'{smb} view /dev/stdin -t {threads} -f bam -S -o - | ' +
'{smb} sort /dev/stdin -t {threads} --tmpdir {tmp_dirpath} -o {bam_fpath}').format(**locals())
run(bwa_cmdline, output_fpath=bam_fpath, stdout_to_outputfile=False)
if dedup:
dedup_bam_fpath = add_suffix(bam_fpath, 'dedup')
dedup_cmdl = '{smb} markdup -t {threads} {bam_fpath} {dedup_bam_fpath}'.format(**locals())
run(dedup_cmdl, output_fpath=dedup_bam_fpath, stdout_to_outputfile=False)
verify_bam(dedup_bam_fpath)
os.rename(dedup_bam_fpath, bam_fpath)
sambamba.index_bam(bam_fpath)
# samtools view -b -S -u - |
# sambamba sort -N -t 8 -m 682M --tmpdir /Molly/saveliev/cancer-dream-syn3/work/align/syn3-normal/split/tx/tmpwdXndE/syn3-normal-sort-1_20000000-sorttmp-full
# -o /Molly/saveliev/cancer-dream-syn3/work/align/syn3-normal/split/tx/tmpwdXndE/syn3-normal-sort-1_20000000.bam
# /dev/stdin
# if dedup:
# info()
# info('Calling SamBlaster to mark duplicates')
# markdup_sam_fpath = markdup_sam(sam_fpath, samblaster)
# if markdup_sam_fpath:
# sam_fpath = markdup_sam_fpath
# info()
# info('Converting to BAM')
# cmdline = sambamba.get_executable() + ' view -t {threads} -S -f bam {sam_fpath}'.format(**locals())
# run(cmdline, output_fpath=bam_fpath, reuse=cfg.reuse_intermediate)
#
# info()
# info('Sorting BAM')
# prefix = splitext(sorted_bam_fpath)[0]
# cmdline = sambamba.get_executable() + ' sort -t {threads} {bam_fpath} -o {sorted_bam_fpath}'.format(**locals())
# run(cmdline, output_fpath=sorted_bam_fpath, stdout_to_outputfile=False, reuse=cfg.reuse_intermediate)
return bam_fpath
def run_multisample_qualimap(output_dir, work_dir, samples, targqc_full_report):
""" 1. Generates Qualimap2 plots and put into plots_dirpath
2. Adds records to targqc_full_report.plots
"""
plots_dirpath = join(output_dir, 'plots')
individual_report_fpaths = [s.qualimap_html_fpath for s in samples]
if isdir(plots_dirpath) and not any(
not can_reuse(join(plots_dirpath, f), individual_report_fpaths)
for f in listdir(plots_dirpath) if not f.startswith('.')):
debug('Qualimap miltisample plots exist - ' + plots_dirpath + ', reusing...')
else:
# Qualimap2 run for multi-sample plots
if len([s.qualimap_html_fpath for s in samples if s.qualimap_html_fpath]) > 0:
if find_executable() is not None: # and get_qualimap_type(find_executable()) == 'full':
qualimap_output_dir = join(work_dir, 'qualimap_multi_bamqc')
_correct_qualimap_genome_results(samples)
_correct_qualimap_insert_size_histogram(samples)
safe_mkdir(qualimap_output_dir)
rows = []
for sample in samples:
if sample.qualimap_html_fpath:
rows += [[sample.name, sample.qualimap_html_fpath]]
data_fpath = write_tsv_rows(([], rows), join(qualimap_output_dir, 'qualimap_results_by_sample.tsv'))
qualimap_plots_dirpath = join(qualimap_output_dir, 'images_multisampleBamQcReport')
cmdline = find_executable() + ' multi-bamqc --data {data_fpath} -outdir {qualimap_output_dir}'.format(**locals())
run(cmdline, env_vars=dict(DISPLAY=None),
checks=[lambda _1, _2: verify_dir(qualimap_output_dir)], reuse=cfg.reuse_intermediate)
if not verify_dir(qualimap_plots_dirpath):
warn('Warning: Qualimap for multi-sample analysis failed to finish. TargQC will not contain plots.')
return None
else:
if exists(plots_dirpath):
shutil.rmtree(plots_dirpath)
shutil.move(qualimap_plots_dirpath, plots_dirpath)
else:
warn('Warning: Qualimap for multi-sample analysis was not found. TargQC will not contain plots.')
return None
targqc_full_report.plots = []
for plot_fpath in listdir(plots_dirpath):
plot_fpath = join(plots_dirpath, plot_fpath)
if verify_file(plot_fpath) and plot_fpath.endswith('.png'):
targqc_full_report.plots.append(relpath(plot_fpath, output_dir))
def determine_sex(work_dir, bam_fpath, avg_depth, genome, target_bed=None):
debug()
debug('Determining sex')
pybedtools.set_tempdir(safe_mkdir(join(work_dir, 'pybedtools_tmp')))
male_bed = None
for k in chry_key_regions_by_genome:
if k in genome:
male_bed = BedTool(chry_key_regions_by_genome.get(k))
break
if not male_bed:
warn('Warning: no male key regions for ' + genome + ', cannot identify sex')
return None
male_area_size = get_total_bed_size(male_bed)
debug('Male region total size: ' + str(male_area_size))
if target_bed:
target_male_bed = join(work_dir, 'male.bed')
with file_transaction(work_dir, target_male_bed) as tx:
BedTool(target_bed).intersect(male_bed).merge().saveas(tx)
target_male_area_size = get_total_bed_size(target_male_bed)
if target_male_area_size == 0:
debug('The male non-PAR region does not overlap with the capture target - cannot determine sex.')
return None
male_bed = target_male_bed
else:
debug('WGS, determining sex based on chrY key regions coverage.')
info('Detecting sex by comparing the Y chromosome key regions coverage and average coverage depth.')
if not bam_fpath:
critical('BAM file is required.')
index_bam(bam_fpath)
chry_mean_coverage = _calc_mean_coverage(work_dir, male_bed, bam_fpath, 1)
debug('Y key regions average depth: ' + str(chry_mean_coverage))
avg_depth = float(avg_depth)
debug('Sample average depth: ' + str(avg_depth))
if avg_depth < AVG_DEPTH_THRESHOLD_TO_DETERMINE_SEX:
debug('Sample average depth is too low (less than ' + str(AVG_DEPTH_THRESHOLD_TO_DETERMINE_SEX) +
') - cannot determine sex')
return None
if chry_mean_coverage == 0:
debug('Y depth is 0 - it\s female')
sex = 'F'
else:
factor = avg_depth / chry_mean_coverage
debug('Sample depth / Y depth = ' + str(factor))
if factor > FEMALE_Y_COVERAGE_FACTOR: # if mean target coverage much higher than chrY coverage
debug('Sample depth is more than ' + str(FEMALE_Y_COVERAGE_FACTOR) + ' times higher than Y depth - it\s female')
sex = 'F'
else:
debug('Sample depth is not more than ' + str(FEMALE_Y_COVERAGE_FACTOR) + ' times higher than Y depth - it\s male')
sex = 'M'
debug('Sex is ' + sex)
debug()
return sex
def parallel_view(n_samples, parallel_cfg, work_dir):
prev_dir = os.getcwd()
os.chdir(safe_mkdir(work_dir))
view = get_parallel_view(n_samples, parallel_cfg)
os.chdir(prev_dir)
try:
yield view
finally:
view.stop()
def get_gender(genome, bam_fpath, bed_fpath, sample, avg_depth):
gender = None
chrom_lengths = ref.get_chrom_lengths(genome)
chrom_names = [chrom for chrom, length in chrom_lengths]
if 'Y' in chrom_names or 'chrY' in chrom_names:
gender = determine_sex(sample.work_dir, bam_fpath, avg_depth, genome, bed_fpath)
if gender:
with open(join(safe_mkdir(sample.dirpath), 'gender.txt'), 'w') as f:
f.write(gender[0].upper())
return gender
def run_qualimap(work_dir, output_dir, output_fpaths, bam_fpath, genome, bed_fpath=None, threads=1):
info('Analysing ' + bam_fpath)
safe_mkdir(dirname(output_dir))
safe_mkdir(output_dir)
mem_cmdl = ''
mem_m = get_qualimap_max_mem(bam_fpath)
mem = str(int(mem_m)) + 'M'
mem_cmdl = '--java-mem-size=' + mem
cmdline = (find_executable() + ' bamqc --skip-duplicated -nt {threads} {mem_cmdl} -nr 5000 '
'-bam {bam_fpath} -outdir {output_dir}')
if genome.startswith('hg') or genome.startswith('GRCh'):
cmdline += ' -gd HUMAN'
if genome.startswith('mm'):
cmdline += ' -gd MOUSE'
if bed_fpath:
cmdline += ' -gff {bed_fpath}'
debug('Using amplicons/capture panel ' + bed_fpath)
cmdline = cmdline.format(**locals())
if not all(can_reuse(fp, [bam_fpath, bed_fpath] if bed_fpath else [bam_fpath]) for fp in output_fpaths):
for fp in output_fpaths:
if isfile(fp):
os.remove(fp)
try:
run(cmdline, env_vars=dict(DISPLAY=None))
except subprocess.CalledProcessError as e:
if 'The alignment file is unsorted.' in e.output:
info()
info('BAM file is unsorted; trying to sort and rerun QualiMap')
sorted_bam_fpath = sort_bam(bam_fpath)
cmdline = cmdline.replace(bam_fpath, sorted_bam_fpath)
run(cmdline, env_vars=dict(DISPLAY=None))
if not all(verify_file(fp, cmp_f=[bam_fpath, bed_fpath] if bed_fpath else [bam_fpath]) for fp in output_fpaths):
critical('Some of the QualiMap results were not generated')
return output_dir
def get_gender(genome, bam_fpath, bed_fpath, sample, avg_depth):
gender = None
chrom_lengths = ref.get_chrom_lengths(genome)
chrom_names = [chrom for chrom, length in chrom_lengths]
if 'Y' in chrom_names or 'chrY' in chrom_names:
gender = determine_sex(sample.work_dir, bam_fpath, avg_depth, genome,
bed_fpath)
if gender:
with open(join(safe_mkdir(sample.dirpath), 'gender.txt'),
'w') as f:
f.write(gender[0].upper())
return gender
def clean_bed(bed_fpath, work_dir):
clean_fpath = intermediate_fname(work_dir, bed_fpath, 'clean')
if not can_reuse(clean_fpath, bed_fpath):
pybedtools.set_tempdir(safe_mkdir(join(work_dir, 'pybedtools_tmp')))
bed = BedTool(bed_fpath)
bed = bed.filter(lambda x: x.chrom and not any(
x.chrom.startswith(e) for e in ['#', ' ', 'track', 'browser']))
bed = bed.remove_invalid()
with file_transaction(work_dir, clean_fpath) as tx_out_file:
bed.saveas(tx_out_file)
verify_bed(clean_fpath, is_critical=True)
debug('Saved clean BED file into ' + clean_fpath)
return clean_fpath
def run_multisample_qualimap(output_dir, work_dir, samples,
targqc_full_report):
""" 1. Generates Qualimap2 plots and put into plots_dirpath
2. Adds records to targqc_full_report.plots
"""
plots_dirpath = join(output_dir, 'plots')
individual_report_fpaths = [s.qualimap_html_fpath for s in samples]
if isdir(plots_dirpath) and not any(
not can_reuse(join(plots_dirpath, f), individual_report_fpaths)
for f in listdir(plots_dirpath) if not f.startswith('.')):
debug('Qualimap miltisample plots exist - ' + plots_dirpath +
', reusing...')
else:
# Qualimap2 run for multi-sample plots
if len(
[s.qualimap_html_fpath
for s in samples if s.qualimap_html_fpath]) > 0:
if find_executable(
) is not None: # and get_qualimap_type(find_executable()) == 'full':
qualimap_output_dir = join(work_dir, 'qualimap_multi_bamqc')
_correct_qualimap_genome_results(samples)
_correct_qualimap_insert_size_histogram(samples)
safe_mkdir(qualimap_output_dir)
rows = []
for sample in samples:
if sample.qualimap_html_fpath:
rows += [[sample.name, sample.qualimap_html_fpath]]
data_fpath = write_tsv_rows(
([], rows),
join(qualimap_output_dir,
'qualimap_results_by_sample.tsv'))
qualimap_plots_dirpath = join(qualimap_output_dir,
'images_multisampleBamQcReport')
cmdline = find_executable(
) + ' multi-bamqc --data {data_fpath} -outdir {qualimap_output_dir}'.format(
**locals())
run(cmdline,
env_vars=dict(DISPLAY=None),
checks=[lambda _1, _2: verify_dir(qualimap_output_dir)],
reuse=cfg.reuse_intermediate)
if not verify_dir(qualimap_plots_dirpath):
warn(
'Warning: Qualimap for multi-sample analysis failed to finish. TargQC will not contain plots.'
)
return None
else:
if exists(plots_dirpath):
shutil.rmtree(plots_dirpath)
shutil.move(qualimap_plots_dirpath, plots_dirpath)
else:
warn(
'Warning: Qualimap for multi-sample analysis was not found. TargQC will not contain plots.'
)
return None
targqc_full_report.plots = []
for plot_fpath in listdir(plots_dirpath):
plot_fpath = join(plots_dirpath, plot_fpath)
if verify_file(plot_fpath) and plot_fpath.endswith('.png'):
targqc_full_report.plots.append(relpath(plot_fpath, output_dir))
def _annotate(bed, ref_bed, chr_order, fai_fpath, work_dir, ori_col_num,
high_confidence=False, reannotate=False, is_debug=False, **kwargs):
# if genome:
# genome_fpath = cut(fai_fpath, 2, output_fpath=intermediate_fname(work_dir, fai_fpath, 'cut2'))
# intersection = bed.intersect(ref_bed, sorted=True, wao=True, g='<(cut -f1,2 ' + fai_fpath + ')')
# intersection = bed.intersect(ref_bed, sorted=True, wao=True, genome=genome.split('-')[0])
# else:
intersection_bed = None
intersection_fpath = None
pybedtools.set_tempdir(safe_mkdir(join(work_dir, 'bedtools')))
if is_debug:
intersection_fpath = join(work_dir, 'intersection.bed')
if isfile(intersection_fpath):
info('Loading from ' + intersection_fpath)
intersection_bed = BedTool(intersection_fpath)
if not intersection_bed:
if count_bed_cols(fai_fpath) == 2:
debug('Fai fields size is 2 ' + fai_fpath)
intersection_bed = bed.intersect(ref_bed, wao=True, g=fai_fpath)
else:
debug('Fai fields is ' + str(count_bed_cols(fai_fpath)) + ', not 2')
intersection_bed = bed.intersect(ref_bed, wao=True)
if is_debug and not isfile(intersection_fpath):
intersection_bed.saveas(intersection_fpath)
debug('Saved intersection to ' + intersection_fpath)
total_annotated = 0
total_uniq_annotated = 0
total_off_target = 0
met = set()
overlaps_by_tx_by_gene_by_loc = OrderedDefaultDict(lambda: OrderedDefaultDict(lambda: defaultdict(list)))
# off_targets = list()
expected_fields_num = ori_col_num + len(ebl.BedCols.cols[:-4]) + 1
for i, intersection_fields in enumerate(intersection_bed):
inters_fields_list = list(intersection_fields)
if len(inters_fields_list) < expected_fields_num:
critical(f'Cannot parse the reference BED file - unexpected number of lines '
'({len(inters_fields_list} in {inters_fields_list}' +
' (less than {expected_fields_num})')
a_chr, a_start, a_end = intersection_fields[:3]
a_extra_columns = intersection_fields[3:ori_col_num]
overlap_fields = [None for _ in ebl.BedCols.cols]
overlap_fields[:len(intersection_fields[ori_col_num:-1])] = intersection_fields[ori_col_num:-1]
keep_gene_column = not reannotate
a_gene = None
if keep_gene_column:
a_gene = a_extra_columns[0]
e_chr = overlap_fields[0]
overlap_size = int(intersection_fields[-1])
assert e_chr == '.' or a_chr == e_chr, f'Error on line {i}: chromosomes don\'t match ({a_chr} vs {e_chr}). Line: {intersection_fields}'
# fs = [None for _ in ebl.BedCols.cols]
# fs[:3] = [a_chr, a_start, a_end]
reg = (a_chr, int(a_start), int(a_end), tuple(a_extra_columns))
if e_chr == '.':
total_off_target += 1
# off_targets.append(fs)
overlaps_by_tx_by_gene_by_loc[reg][a_gene] = OrderedDefaultDict(list)
else:
# fs[3:-1] = db_feature_fields[3:-1]
total_annotated += 1
if (a_chr, a_start, a_end) not in met:
total_uniq_annotated += 1
met.add((a_chr, a_start, a_end))
e_gene = overlap_fields[ebl.BedCols.GENE]
if keep_gene_column and e_gene != a_gene:
overlaps_by_tx_by_gene_by_loc[reg][a_gene] = OrderedDefaultDict(list)
else:
transcript_id = overlap_fields[ebl.BedCols.ENSEMBL_ID]
overlaps_by_tx_by_gene_by_loc[reg][e_gene][transcript_id].append((overlap_fields, overlap_size))
info(' Total annotated regions: ' + str(total_annotated))
info(' Total unique annotated regions: ' + str(total_uniq_annotated))
info(' Total off target regions: ' + str(total_off_target))
info('Resolving ambiguities...')
annotated = _resolve_ambiguities(overlaps_by_tx_by_gene_by_loc, chr_order, **kwargs)
return annotated
def proc_fastq(samples, parall_view, work_dir, bwa_prefix, downsample_to, num_pairs_by_sample=None, dedup=True):
num_pairs_by_sample = num_pairs_by_sample or dict()
if downsample_to:
# Read pairs counts
debug()
if all(s.name in num_pairs_by_sample for s in samples):
debug('Using read pairs counts extracted from FastQC reports')
elif all(can_reuse(make_pair_counts_fpath(join(work_dir, s.name)), s.l_fpath) for s in samples):
debug('Reusing pairs counts, reading from files')
num_pairs_by_sample = {s.name: int(open(make_pair_counts_fpath(join(work_dir, s.name))).read().strip()) for s in samples}
else:
info('Counting read pairs')
num_pairs = parall_view.run(count_read_pairs, [[s.name, safe_mkdir(join(work_dir, s.name)), s.l_fpath] for s in samples])
num_pairs_by_sample = {s.name: pairs_count for s, pairs_count in zip(samples, num_pairs)}
# Downsampling
debug()
if all(can_reuse(make_downsampled_fpath(join(work_dir, s.name), s.l_fpath), s.l_fpath) and
can_reuse(make_downsampled_fpath(join(work_dir, s.name), s.r_fpath), s.r_fpath) for s in samples):
debug('Reusing downsampled FastQ')
for s in samples:
s.l_fpath = make_downsampled_fpath(join(work_dir, s.name), s.l_fpath)
s.r_fpath = make_downsampled_fpath(join(work_dir, s.name), s.r_fpath)
else:
if isinstance(downsample_to, float):
info('Downsampling FastQ to ' + str(float(downsample_to)) + ' fraction of reads')
else:
info('Downsampling FastQ to ' + str(int(downsample_to)) + ' read pairs')
fastq_pairs = parall_view.run(downsample,
[[join(work_dir, s.name), s.name, s.l_fpath, s.r_fpath, downsample_to, num_pairs_by_sample.get(s.name)]
for s in samples])
for s, (l_r, r_r) in zip(samples, fastq_pairs):
s.l_fpath = l_r
s.r_fpath = r_r
else:
info('Skipping downsampling')
debug()
if all(can_reuse(make_bam_fpath(join(work_dir, s.name)), [s.l_fpath, s.r_fpath]) for s in samples):
debug('All downsampled BAM exists, reusing')
for s in samples:
s.bam = make_bam_fpath(join(work_dir, s.name))
else:
bwa = which('bwa')
if not isfile(bwa):
critical('BWA not found under ' + bwa)
smb = sambamba.get_executable()
if not (bwa and smb):
if not bwa: err('Error: bwa is required for the alignment pipeline')
if not smb: err('Error: sambamba is required for the alignment pipeline')
critical('Tools required for alignment not found')
info('Aligning reads to the reference')
bam_fpaths = parall_view.run(align,
[[join(work_dir, s.name), s.name, s.l_fpath, s.r_fpath, bwa, smb, bwa_prefix, dedup, parall_view.cores_per_job]
for s in samples])
bam_fpaths = [verify_bam(b) for b in bam_fpaths]
if len(bam_fpaths) < len(samples):
critical('Some samples were not aligned successfully.')
for bam, s in zip(bam_fpaths, samples):
s.bam = bam
return num_pairs_by_sample
def annotate(input_bed_fpath, output_fpath, work_dir, genome=None,
reannotate=True, high_confidence=False, only_canonical=False,
coding_only=False, short=False, extended=False, is_debug=False, **kwargs):
debug('Getting features from storage')
features_bed = ebl.get_all_features(genome)
if features_bed is None:
critical('Genome ' + genome + ' is not supported. Supported: ' + ', '.join(ebl.SUPPORTED_GENOMES))
if genome:
fai_fpath = reference_data.get_fai(genome)
chr_order = reference_data.get_chrom_order(genome)
else:
fai_fpath = None
chr_order = bed_chrom_order(input_bed_fpath)
input_bed_fpath = sort_bed(input_bed_fpath, work_dir=work_dir, chr_order=chr_order, genome=genome)
ori_bed = BedTool(input_bed_fpath)
ori_col_num = ori_bed.field_count()
reannotate = reannotate or ori_col_num == 3
pybedtools.set_tempdir(safe_mkdir(join(work_dir, 'bedtools')))
ori_bed = BedTool(input_bed_fpath)
# if reannotate:
# bed = BedTool(input_bed_fpath).cut([0, 1, 2])
# keep_gene_column = False
# else:
# if col_num > 4:
# bed = BedTool(input_bed_fpath).cut([0, 1, 2, 3])
# keep_gene_column = True
# features_bed = features_bed.saveas()
# cols = features_bed.field_count()
# if cols < 12:
# features_bed = features_bed.each(lambda f: f + ['.']*(12-cols))
if high_confidence:
features_bed = features_bed.filter(ebl.high_confidence_filter)
if only_canonical:
features_bed = features_bed.filter(ebl.get_only_canonical_filter(genome))
if coding_only:
features_bed = features_bed.filter(ebl.protein_coding_filter)
# unique_tx_by_gene = find_best_tx_by_gene(features_bed)
info('Extracting features from Ensembl GTF')
features_bed = features_bed.filter(lambda x:
x[ebl.BedCols.FEATURE] in ['exon', 'CDS', 'stop_codon', 'transcript'])
# x[ebl.BedCols.ENSEMBL_ID] == unique_tx_by_gene[x[ebl.BedCols.GENE]])
info('Overlapping regions with Ensembl data')
if is_debug:
ori_bed = ori_bed.saveas(join(work_dir, 'bed.bed'))
debug(f'Saved regions to {ori_bed.fn}')
features_bed = features_bed.saveas(join(work_dir, 'features.bed'))
debug(f'Saved features to {features_bed.fn}')
annotated = _annotate(ori_bed, features_bed, chr_order, fai_fpath, work_dir, ori_col_num,
high_confidence=False, reannotate=reannotate, is_debug=is_debug, **kwargs)
full_header = [ebl.BedCols.names[i] for i in ebl.BedCols.cols]
add_ori_extra_fields = ori_col_num > 3
if not reannotate and ori_col_num == 4:
add_ori_extra_fields = False # no need to report the original gene field if we are not re-annotating
info('Saving annotated regions...')
total = 0
with file_transaction(work_dir, output_fpath) as tx:
with open(tx, 'w') as out:
header = full_header[:6]
if short:
header = full_header[:4]
if extended:
header = full_header[:-1]
if add_ori_extra_fields:
header.append(full_header[-1])
if extended:
out.write('## ' + ebl.BedCols.names[ebl.BedCols.TX_OVERLAP_PERCENTAGE] +
': part of region overlapping with transcripts\n')
out.write('## ' + ebl.BedCols.names[ebl.BedCols.EXON_OVERLAPS_PERCENTAGE] +
': part of region overlapping with exons\n')
out.write('## ' + ebl.BedCols.names[ebl.BedCols.CDS_OVERLAPS_PERCENTAGE] +
': part of region overlapping with protein coding regions\n')
out.write('\t'.join(header) + '\n')
for full_fields in annotated:
fields = full_fields[:6]
if short:
fields = full_fields[:4]
if extended:
fields = full_fields[:-1]
if add_ori_extra_fields:
fields.append(full_fields[-1])
out.write('\t'.join(map(_format_field, fields)) + '\n')
total += 1
debug('Saved ' + str(total) + ' total annotated regions')
return output_fpath
def annotate(input_bed_fpath,
output_fpath,
work_dir,
genome=None,
reannotate=True,
high_confidence=False,
only_canonical=False,
coding_only=False,
short=False,
extended=False,
is_debug=False,
**kwargs):
debug('Getting features from storage')
features_bed = ebl.get_all_features(genome)
if features_bed is None:
critical('Genome ' + genome + ' is not supported. Supported: ' +
', '.join(ebl.SUPPORTED_GENOMES))
if genome:
fai_fpath = reference_data.get_fai(genome)
chr_order = reference_data.get_chrom_order(genome)
else:
fai_fpath = None
chr_order = bed_chrom_order(input_bed_fpath)
input_bed_fpath = sort_bed(input_bed_fpath,
work_dir=work_dir,
chr_order=chr_order,
genome=genome)
ori_bed = BedTool(input_bed_fpath)
ori_col_num = ori_bed.field_count()
reannotate = reannotate or ori_col_num == 3
pybedtools.set_tempdir(safe_mkdir(join(work_dir, 'bedtools')))
ori_bed = BedTool(input_bed_fpath)
# if reannotate:
# bed = BedTool(input_bed_fpath).cut([0, 1, 2])
# keep_gene_column = False
# else:
# if col_num > 4:
# bed = BedTool(input_bed_fpath).cut([0, 1, 2, 3])
# keep_gene_column = True
# features_bed = features_bed.saveas()
# cols = features_bed.field_count()
# if cols < 12:
# features_bed = features_bed.each(lambda f: f + ['.']*(12-cols))
if high_confidence:
features_bed = features_bed.filter(ebl.high_confidence_filter)
if only_canonical:
features_bed = features_bed.filter(
ebl.get_only_canonical_filter(genome))
if coding_only:
features_bed = features_bed.filter(ebl.protein_coding_filter)
# unique_tx_by_gene = find_best_tx_by_gene(features_bed)
info('Extracting features from Ensembl GTF')
features_bed = features_bed.filter(lambda x: x[
ebl.BedCols.FEATURE] in ['exon', 'CDS', 'stop_codon', 'transcript'])
# x[ebl.BedCols.ENSEMBL_ID] == unique_tx_by_gene[x[ebl.BedCols.GENE]])
info('Overlapping regions with Ensembl data')
if is_debug:
ori_bed = ori_bed.saveas(join(work_dir, 'bed.bed'))
debug(f'Saved regions to {ori_bed.fn}')
features_bed = features_bed.saveas(join(work_dir, 'features.bed'))
debug(f'Saved features to {features_bed.fn}')
annotated = _annotate(ori_bed,
features_bed,
chr_order,
fai_fpath,
work_dir,
ori_col_num,
high_confidence=False,
reannotate=reannotate,
is_debug=is_debug,
**kwargs)
full_header = [ebl.BedCols.names[i] for i in ebl.BedCols.cols]
add_ori_extra_fields = ori_col_num > 3
if not reannotate and ori_col_num == 4:
add_ori_extra_fields = False # no need to report the original gene field if we are not re-annotating
info('Saving annotated regions...')
total = 0
with file_transaction(work_dir, output_fpath) as tx:
with open(tx, 'w') as out:
header = full_header[:6]
if short:
header = full_header[:4]
if extended:
header = full_header[:-1]
if add_ori_extra_fields:
header.append(full_header[-1])
if extended:
out.write(
'## ' +
ebl.BedCols.names[ebl.BedCols.TX_OVERLAP_PERCENTAGE] +
': part of region overlapping with transcripts\n')
out.write(
'## ' +
ebl.BedCols.names[ebl.BedCols.EXON_OVERLAPS_PERCENTAGE] +
': part of region overlapping with exons\n')
out.write(
'## ' +
ebl.BedCols.names[ebl.BedCols.CDS_OVERLAPS_PERCENTAGE] +
': part of region overlapping with protein coding regions\n'
)
out.write('\t'.join(header) + '\n')
for full_fields in annotated:
fields = full_fields[:6]
if short:
fields = full_fields[:4]
if extended:
fields = full_fields[:-1]
if add_ori_extra_fields:
fields.append(full_fields[-1])
out.write('\t'.join(map(_format_field, fields)) + '\n')
total += 1
debug('Saved ' + str(total) + ' total annotated regions')
return output_fpath
def start_targqc(
work_dir,
output_dir,
samples,
target_bed_fpath,
parallel_cfg,
bwa_prefix,
fai_fpath=None,
genome=config.genome,
depth_threshs=config.depth_thresholds,
downsample_to=config.downsample_fraction,
padding=config.padding,
dedup=config.dedup,
num_pairs_by_sample=None,
reannotate=config.reannotate,
):
d = get_description()
info('*' * len(d))
info(d)
info('*' * len(d))
info()
fai_fpath = fai_fpath or ref.get_fai(genome)
target = Target(work_dir,
output_dir,
fai_fpath,
padding=padding,
bed_fpath=target_bed_fpath,
reannotate=reannotate,
genome=genome,
is_debug=logger.is_debug)
fastq_samples = [
s for s in samples if not s.bam and s.l_fpath and s.r_fpath
]
from targqc.utilz.parallel import parallel_view
if fastq_samples:
if not bwa_prefix:
critical('--bwa-prefix is required when running from fastq')
with parallel_view(len(fastq_samples), parallel_cfg,
join(work_dir, 'sge_fastq')) as view:
num_pairs_by_sample = proc_fastq(fastq_samples,
view,
work_dir,
bwa_prefix,
downsample_to,
num_pairs_by_sample,
dedup=dedup)
info()
for s in samples:
if s.bam:
info(s.name + ': using alignment ' + s.bam)
with parallel_view(len(samples), parallel_cfg, join(work_dir,
'sge_bam')) as view:
info('Sorting BAMs...')
sorted_bams = view.run(
sort_bam,
[[s.bam, safe_mkdir(join(work_dir, s.name))] for s in samples])
for s, sorted_bam in zip(samples, sorted_bams):
s.bam = sorted_bam
if all(can_reuse(s.bam + '.bai', s.bam) for s in samples):
debug('BAM indexes exists')
else:
info('Indexing BAMs...')
view.run(index_bam, [[s.bam] for s in samples])
info('Making general reports...')
make_general_reports(view,
samples,
target,
genome,
depth_threshs,
padding,
num_pairs_by_sample,
is_debug=logger.is_debug,
reannotate=reannotate,
fai_fpath=fai_fpath)
info()
info('*' * 70)
tsv_fpath, html_fpath = make_tarqc_html_report(output_dir,
work_dir,
samples,
bed_fpath=target_bed_fpath)
info('TargQC summary saved in: ')
info(' ' + html_fpath)
info(' ' + tsv_fpath)
info()
with parallel_view(len(samples), parallel_cfg, join(work_dir,
'sge_bam')) as view:
info('Making region-level reports...')
make_region_reports(view, work_dir, samples, target, genome,
depth_threshs)
info()
info('*' * 70)
tsv_region_rep_fpath = combined_regional_reports(work_dir, output_dir,
samples)
info()
info('*' * 70)
info('TargQC summary saved in: ')
info(' ' + html_fpath)
info(' ' + tsv_fpath)
info('Per-region coverage statistics saved into:')
info(' ' + tsv_region_rep_fpath)
return html_fpath
def proc_fastq(samples,
parall_view,
work_dir,
bwa_prefix,
downsample_to,
num_pairs_by_sample=None,
dedup=True):
num_pairs_by_sample = num_pairs_by_sample or dict()
if downsample_to:
# Read pairs counts
debug()
if all(s.name in num_pairs_by_sample for s in samples):
debug('Using read pairs counts extracted from FastQC reports')
elif all(
can_reuse(make_pair_counts_fpath(join(work_dir, s.name)),
s.l_fpath) for s in samples):
debug('Reusing pairs counts, reading from files')
num_pairs_by_sample = {
s.name: int(
open(make_pair_counts_fpath(join(work_dir,
s.name))).read().strip())
for s in samples
}
else:
info('Counting read pairs')
num_pairs = parall_view.run(
count_read_pairs,
[[s.name,
safe_mkdir(join(work_dir, s.name)), s.l_fpath]
for s in samples])
num_pairs_by_sample = {
s.name: pairs_count
for s, pairs_count in zip(samples, num_pairs)
}
# Downsampling
debug()
if all(
can_reuse(
make_downsampled_fpath(join(work_dir, s.name), s.l_fpath),
s.l_fpath) and can_reuse(
make_downsampled_fpath(join(work_dir, s.name),
s.r_fpath), s.r_fpath)
for s in samples):
debug('Reusing downsampled FastQ')
for s in samples:
s.l_fpath = make_downsampled_fpath(join(work_dir, s.name),
s.l_fpath)
s.r_fpath = make_downsampled_fpath(join(work_dir, s.name),
s.r_fpath)
else:
if isinstance(downsample_to, float):
info('Downsampling FastQ to ' + str(float(downsample_to)) +
' fraction of reads')
else:
info('Downsampling FastQ to ' + str(int(downsample_to)) +
' read pairs')
fastq_pairs = parall_view.run(downsample, [[
join(work_dir, s.name), s.name, s.l_fpath, s.r_fpath,
downsample_to,
num_pairs_by_sample.get(s.name)
] for s in samples])
for s, (l_r, r_r) in zip(samples, fastq_pairs):
s.l_fpath = l_r
s.r_fpath = r_r
else:
info('Skipping downsampling')
debug()
if all(
can_reuse(make_bam_fpath(join(work_dir, s.name)),
[s.l_fpath, s.r_fpath]) for s in samples):
debug('All downsampled BAM exists, reusing')
for s in samples:
s.bam = make_bam_fpath(join(work_dir, s.name))
else:
bwa = which('bwa')
if not isfile(bwa):
critical('BWA not found under ' + bwa)
smb = sambamba.get_executable()
if not (bwa and smb):
if not bwa:
err('Error: bwa is required for the alignment pipeline')
if not smb:
err('Error: sambamba is required for the alignment pipeline')
critical('Tools required for alignment not found')
info('Aligning reads to the reference')
bam_fpaths = parall_view.run(align, [[
join(work_dir, s.name), s.name, s.l_fpath, s.r_fpath, bwa, smb,
bwa_prefix, dedup, parall_view.cores_per_job
] for s in samples])
bam_fpaths = [verify_bam(b) for b in bam_fpaths]
if len(bam_fpaths) < len(samples):
critical('Some samples were not aligned successfully.')
for bam, s in zip(bam_fpaths, samples):
s.bam = bam
return num_pairs_by_sample
def _annotate(bed,
ref_bed,
chr_order,
fai_fpath,
work_dir,
ori_col_num,
high_confidence=False,
reannotate=False,
is_debug=False,
**kwargs):
# if genome:
# genome_fpath = cut(fai_fpath, 2, output_fpath=intermediate_fname(work_dir, fai_fpath, 'cut2'))
# intersection = bed.intersect(ref_bed, sorted=True, wao=True, g='<(cut -f1,2 ' + fai_fpath + ')')
# intersection = bed.intersect(ref_bed, sorted=True, wao=True, genome=genome.split('-')[0])
# else:
intersection_bed = None
intersection_fpath = None
pybedtools.set_tempdir(safe_mkdir(join(work_dir, 'bedtools')))
if is_debug:
intersection_fpath = join(work_dir, 'intersection.bed')
if isfile(intersection_fpath):
info('Loading from ' + intersection_fpath)
intersection_bed = BedTool(intersection_fpath)
if not intersection_bed:
if count_bed_cols(fai_fpath) == 2:
debug('Fai fields size is 2 ' + fai_fpath)
intersection_bed = bed.intersect(ref_bed, wao=True, g=fai_fpath)
else:
debug('Fai fields is ' + str(count_bed_cols(fai_fpath)) +
', not 2')
intersection_bed = bed.intersect(ref_bed, wao=True)
if is_debug and not isfile(intersection_fpath):
intersection_bed.saveas(intersection_fpath)
debug('Saved intersection to ' + intersection_fpath)
total_annotated = 0
total_uniq_annotated = 0
total_off_target = 0
met = set()
overlaps_by_tx_by_gene_by_loc = OrderedDefaultDict(
lambda: OrderedDefaultDict(lambda: defaultdict(list)))
# off_targets = list()
expected_fields_num = ori_col_num + len(ebl.BedCols.cols[:-4]) + 1
for i, intersection_fields in enumerate(intersection_bed):
inters_fields_list = list(intersection_fields)
if len(inters_fields_list) < expected_fields_num:
critical(
f'Cannot parse the reference BED file - unexpected number of lines '
'({len(inters_fields_list} in {inters_fields_list}' +
' (less than {expected_fields_num})')
a_chr, a_start, a_end = intersection_fields[:3]
a_extra_columns = intersection_fields[3:ori_col_num]
overlap_fields = [None for _ in ebl.BedCols.cols]
overlap_fields[:len(intersection_fields[ori_col_num:-1]
)] = intersection_fields[ori_col_num:-1]
keep_gene_column = not reannotate
a_gene = None
if keep_gene_column:
a_gene = a_extra_columns[0]
e_chr = overlap_fields[0]
overlap_size = int(intersection_fields[-1])
assert e_chr == '.' or a_chr == e_chr, f'Error on line {i}: chromosomes don\'t match ({a_chr} vs {e_chr}). Line: {intersection_fields}'
# fs = [None for _ in ebl.BedCols.cols]
# fs[:3] = [a_chr, a_start, a_end]
reg = (a_chr, int(a_start), int(a_end), tuple(a_extra_columns))
if e_chr == '.':
total_off_target += 1
# off_targets.append(fs)
overlaps_by_tx_by_gene_by_loc[reg][a_gene] = OrderedDefaultDict(
list)
else:
# fs[3:-1] = db_feature_fields[3:-1]
total_annotated += 1
if (a_chr, a_start, a_end) not in met:
total_uniq_annotated += 1
met.add((a_chr, a_start, a_end))
e_gene = overlap_fields[ebl.BedCols.GENE]
if keep_gene_column and e_gene != a_gene:
overlaps_by_tx_by_gene_by_loc[reg][
a_gene] = OrderedDefaultDict(list)
else:
transcript_id = overlap_fields[ebl.BedCols.ENSEMBL_ID]
overlaps_by_tx_by_gene_by_loc[reg][e_gene][
transcript_id].append((overlap_fields, overlap_size))
info(' Total annotated regions: ' + str(total_annotated))
info(' Total unique annotated regions: ' + str(total_uniq_annotated))
info(' Total off target regions: ' + str(total_off_target))
info('Resolving ambiguities...')
annotated = _resolve_ambiguities(overlaps_by_tx_by_gene_by_loc, chr_order,
**kwargs)
return annotated
def determine_sex(work_dir, bam_fpath, avg_depth, genome, target_bed=None):
debug()
debug('Determining sex')
pybedtools.set_tempdir(safe_mkdir(join(work_dir, 'pybedtools_tmp')))
male_bed = None
for k in chry_key_regions_by_genome:
if k in genome:
male_bed = BedTool(chry_key_regions_by_genome.get(k))
break
if not male_bed:
warn('Warning: no male key regions for ' + genome +
', cannot identify sex')
return None
male_area_size = get_total_bed_size(male_bed)
debug('Male region total size: ' + str(male_area_size))
if target_bed:
target_male_bed = join(work_dir, 'male.bed')
with file_transaction(work_dir, target_male_bed) as tx:
BedTool(target_bed).intersect(male_bed).merge().saveas(tx)
target_male_area_size = get_total_bed_size(target_male_bed)
if target_male_area_size == 0:
debug(
'The male non-PAR region does not overlap with the capture target - cannot determine sex.'
)
return None
male_bed = target_male_bed
else:
debug('WGS, determining sex based on chrY key regions coverage.')
info(
'Detecting sex by comparing the Y chromosome key regions coverage and average coverage depth.'
)
if not bam_fpath:
critical('BAM file is required.')
index_bam(bam_fpath)
chry_mean_coverage = _calc_mean_coverage(work_dir, male_bed, bam_fpath, 1)
debug('Y key regions average depth: ' + str(chry_mean_coverage))
avg_depth = float(avg_depth)
debug('Sample average depth: ' + str(avg_depth))
if avg_depth < AVG_DEPTH_THRESHOLD_TO_DETERMINE_SEX:
debug('Sample average depth is too low (less than ' +
str(AVG_DEPTH_THRESHOLD_TO_DETERMINE_SEX) +
') - cannot determine sex')
return None
if chry_mean_coverage == 0:
debug('Y depth is 0 - it\s female')
sex = 'F'
else:
factor = avg_depth / chry_mean_coverage
debug('Sample depth / Y depth = ' + str(factor))
if factor > FEMALE_Y_COVERAGE_FACTOR: # if mean target coverage much higher than chrY coverage
debug('Sample depth is more than ' +
str(FEMALE_Y_COVERAGE_FACTOR) +
' times higher than Y depth - it\s female')
sex = 'F'
else:
debug('Sample depth is not more than ' +
str(FEMALE_Y_COVERAGE_FACTOR) +
' times higher than Y depth - it\s male')
sex = 'M'
debug('Sex is ' + sex)
debug()
return sex
def get_total_bed_size(bed_fpath, work_dir=None):
if work_dir:
pybedtools.set_tempdir(safe_mkdir(join(work_dir, 'pybedtools_tmp')))
return sum(len(x) for x in BedTool(bed_fpath).merge())