def open_gzipsafe(f, mode='r'):
# mode_t = mode.replace('b', '')
# mode_b = mode if 'b' in mode else mode + 'b'
if f.endswith('.gz') or f.endswith('.gzip') or f.endswith(
'.gz.tx') or f.endswith('.gzip.tx'):
try:
h = gzip.open(f, mode=mode + 't')
except IOError as e:
err('Error opening gzip ' + f + ': ' + str(e) +
', opening as plain text')
return open(f, mode=mode)
else:
if 'w' in mode:
return h
else:
try:
h.read(1)
except IOError as e:
err('Error opening gzip ' + f + ': ' + str(e) +
', opening as plain text')
h.close()
return open(f, mode=mode)
else:
h.close()
h = gzip.open(f, mode=mode + 't')
return h
else:
return open(f, mode=mode)
def bgzip_and_tabix(fpath, reuse=False, tabix_parameters='', **kwargs):
gzipped_fpath = join(fpath + '.gz')
tbi_fpath = gzipped_fpath + '.tbi'
if reuse and \
file_exists(gzipped_fpath) and (getctime(gzipped_fpath) >= getctime(fpath) if file_exists(fpath) else True) and \
file_exists(tbi_fpath) and getctime(tbi_fpath) >= getctime(gzipped_fpath):
info('Actual compressed file and index exist, reusing')
return gzipped_fpath
info('Compressing and tabixing file, writing ' + gzipped_fpath + '(.tbi)')
bgzip = which('bgzip')
tabix = which('tabix')
if not bgzip:
err('Cannot index file because bgzip is not found')
if not tabix:
err('Cannot index file because tabix is not found')
if not bgzip and not tabix:
return fpath
if isfile(gzipped_fpath):
os.remove(gzipped_fpath)
if isfile(tbi_fpath):
os.remove(tbi_fpath)
info('BGzipping ' + fpath)
cmdline = '{bgzip} {fpath}'.format(**locals())
call_process.run(cmdline)
info('Tabixing ' + gzipped_fpath)
cmdline = '{tabix} {tabix_parameters} {gzipped_fpath}'.format(**locals())
call_process.run(cmdline)
return gzipped_fpath
def file_exists_check(output_fpath=None, input_fpath=None):
if output_fpath is None:
return True
ok = os.path.exists(output_fpath)
if not ok:
err("Did not find output file {0}".format(output_fpath))
return ok
def file_nonempty_check(output_fpath=None, input_fpath=None):
if output_fpath is None:
return True
ok = file_exists_check(output_fpath)
if not ok:
err("Did not find non-empty output file {0}".format(output_fpath))
return ok
def open_gzipsafe(f, mode='r'):
# mode_t = mode.replace('b', '')
# mode_b = mode if 'b' in mode else mode + 'b'
if f.endswith('.gz') or f.endswith('.gzip') or f.endswith('.gz.tx') or f.endswith('.gzip.tx'):
try:
h = gzip.open(f, mode=mode + 't')
except IOError as e:
err('Error opening gzip ' + f + ': ' + str(e) + ', opening as plain text')
return open(f, mode=mode)
else:
if 'w' in mode:
return h
else:
try:
h.read(1)
except IOError as e:
err('Error opening gzip ' + f + ': ' + str(e) + ', opening as plain text')
h.close()
return open(f, mode=mode)
else:
h.close()
h = gzip.open(f, mode=mode + 't')
return h
else:
return open(f, mode=mode)
def count_bed_cols(bed_fpath):
with open(bed_fpath) as f:
for l in f:
if l and l.strip() and not l.startswith('#'):
return len(l.split('\t'))
# return len(next(dropwhile(lambda x: x.strip().startswith('#'), open(bed_fpath))).split('\t'))
err('Empty bed file: ' + bed_fpath)
return None
def load_yaml_config(fpath):
verify_file(fpath, is_critical=True)
try:
dic = load_yaml(open(fpath))
except Exception:
err(format_exc())
critical('Could not parse bcbio YAML ' + fpath)
else:
return dic
def load_yaml_config(fpath):
verify_file(fpath, is_critical=True)
try:
dic = load_yaml(open(fpath))
except Exception:
err(format_exc())
critical('Could not parse bcbio YAML ' + fpath)
else:
return dic
def bam_to_bed_nocnf(bam_fpath, bedtools='bedtools', gzip='gzip'):
info(
'Converting the BAM to BED to save some memory.'
) # from here: http://davetang.org/muse/2015/08/05/creating-a-coverage-plot-using-bedtools-and-r/
bam_bed_fpath = splitext_plus(bam_fpath)[0] + '.bed.gz'
cmdline = '{bedtools} bamtobed -i {bam_fpath} | {gzip} > {bam_bed_fpath}'.format(
**locals())
info(cmdline)
os.system(cmdline)
bam_bed_fpath = verify_file(bam_bed_fpath)
if bam_bed_fpath:
info('Done, saved to ' + bam_bed_fpath)
else:
err('Error, result is non-existent or empty')
return bam_bed_fpath
def find_fastq_pairs(fpaths):
info('Finding FastQ pairs...')
fastqs_by_sample_name = dict()
for fpath in fpaths:
fn, ext = splitext_plus(basename(fpath))
if ext in ['.fq', '.fq.gz', '.fastq', '.fastq.gz']:
sname, l_fpath, r_fpath = None, None, None
if fn.endswith('_1'):
sname = fn[:-2]
l_fpath = fpath
if fn.endswith('_R1'):
sname = fn[:-3]
l_fpath = fpath
if fn.endswith('_2'):
sname = fn[:-2]
r_fpath = fpath
if fn.endswith('_R2'):
sname = fn[:-3]
r_fpath = fpath
if sname:
m = re.match(r'(.*)_S\d+', sname)
if m:
sname = m.group(1)
sname = sname.replace('-', '_')
else:
sname = fn
info('Cannot detect file for ' + sname)
l, r = fastqs_by_sample_name.get(sname, (None, None))
if l and l_fpath:
critical('Duplicated left FastQ files for ' + sname + ': ' +
l + ' and ' + l_fpath)
if r and r_fpath:
critical('Duplicated right FastQ files for ' + sname + ': ' +
r + ' and ' + r_fpath)
fastqs_by_sample_name[sname] = l or l_fpath, r or r_fpath
fixed_fastqs_by_sample_name = dict()
for sname, (l, r) in fastqs_by_sample_name.items():
if not l:
err('ERROR: for sample ' + sname + ', left reads not found')
if not r:
err('ERROR: for sample ' + sname + ', right reads not found')
if l and r:
fixed_fastqs_by_sample_name[sname] = l, r
return fixed_fastqs_by_sample_name
def run(self, fn, param_lists):
if self.n_samples == 0:
return []
assert self.n_samples == len(param_lists)
n_params = len(param_lists[0])
for sample_i, params in enumerate(param_lists):
if params is None:
err('Parameter list for sample ' + str(sample_i) + ' is None')
if len(params) != n_params:
err('Parameter list for sample ' + str(sample_i) + ' (' +
str(len(params)) +
') does not equal to the one for sample 1 (' +
str(n_params) + ')')
res = self._view.view.map(
fn,
*([params[param_i] for params in param_lists]
for param_i in range(n_params)))
return res
def file_reasonable_size(output_fpath, input_fpath):
ok = file_exists_check(output_fpath)
if not ok:
return ok
# named pipes -- we can't calculate size
if input_fpath.strip().startswith("<("):
return True
if input_fpath.endswith((".bam", ".gz")):
scale = 7.0
else:
scale = 10.0
orig_size = os.path.getsize(input_fpath) / pow(1024.0, 3)
out_size = os.path.getsize(output_fpath) / pow(1024.0, 3)
if out_size < (orig_size / scale):
err("Output file unexpectedly small. %.1fGb for output versus "
"%.1fGb for the input file. This often indicates a truncated "
"BAM file or memory errors during the run." % (out_size, orig_size))
return False
else:
return True
def proc_fastq(samples,
parall_view,
work_dir,
bwa_prefix,
downsample_to,
num_pairs_by_sample=None,
dedup=True):
num_pairs_by_sample = num_pairs_by_sample or dict()
if downsample_to:
# Read pairs counts
debug()
if all(s.name in num_pairs_by_sample for s in samples):
debug('Using read pairs counts extracted from FastQC reports')
elif all(
can_reuse(make_pair_counts_fpath(join(work_dir, s.name)),
s.l_fpath) for s in samples):
debug('Reusing pairs counts, reading from files')
num_pairs_by_sample = {
s.name: int(
open(make_pair_counts_fpath(join(work_dir,
s.name))).read().strip())
for s in samples
}
else:
info('Counting read pairs')
num_pairs = parall_view.run(
count_read_pairs,
[[s.name,
safe_mkdir(join(work_dir, s.name)), s.l_fpath]
for s in samples])
num_pairs_by_sample = {
s.name: pairs_count
for s, pairs_count in zip(samples, num_pairs)
}
# Downsampling
debug()
if all(
can_reuse(
make_downsampled_fpath(join(work_dir, s.name), s.l_fpath),
s.l_fpath) and can_reuse(
make_downsampled_fpath(join(work_dir, s.name),
s.r_fpath), s.r_fpath)
for s in samples):
debug('Reusing downsampled FastQ')
for s in samples:
s.l_fpath = make_downsampled_fpath(join(work_dir, s.name),
s.l_fpath)
s.r_fpath = make_downsampled_fpath(join(work_dir, s.name),
s.r_fpath)
else:
if isinstance(downsample_to, float):
info('Downsampling FastQ to ' + str(float(downsample_to)) +
' fraction of reads')
else:
info('Downsampling FastQ to ' + str(int(downsample_to)) +
' read pairs')
fastq_pairs = parall_view.run(downsample, [[
join(work_dir, s.name), s.name, s.l_fpath, s.r_fpath,
downsample_to,
num_pairs_by_sample.get(s.name)
] for s in samples])
for s, (l_r, r_r) in zip(samples, fastq_pairs):
s.l_fpath = l_r
s.r_fpath = r_r
else:
info('Skipping downsampling')
debug()
if all(
can_reuse(make_bam_fpath(join(work_dir, s.name)),
[s.l_fpath, s.r_fpath]) for s in samples):
debug('All downsampled BAM exists, reusing')
for s in samples:
s.bam = make_bam_fpath(join(work_dir, s.name))
else:
bwa = which('bwa')
if not isfile(bwa):
critical('BWA not found under ' + bwa)
smb = sambamba.get_executable()
if not (bwa and smb):
if not bwa:
err('Error: bwa is required for the alignment pipeline')
if not smb:
err('Error: sambamba is required for the alignment pipeline')
critical('Tools required for alignment not found')
info('Aligning reads to the reference')
bam_fpaths = parall_view.run(align, [[
join(work_dir, s.name), s.name, s.l_fpath, s.r_fpath, bwa, smb,
bwa_prefix, dedup, parall_view.cores_per_job
] for s in samples])
bam_fpaths = [verify_bam(b) for b in bam_fpaths]
if len(bam_fpaths) < len(samples):
critical('Some samples were not aligned successfully.')
for bam, s in zip(bam_fpaths, samples):
s.bam = bam
return num_pairs_by_sample
def _save_best_details_for_each_gene(depth_threshs, samples, output_dir):
metric_storage = get_detailed_metric_storage(depth_threshs)
report = PerRegionSampleReport(sample='Best',
metric_storage=metric_storage)
report.add_record(
'Sample', 'contains best values from all samples: ' +
', '.join([s.name for s in samples]))
total_regions = 0
fpaths = [
s.targqc_region_tsv for s in samples
if verify_file(s.targqc_region_tsv)
]
if not fpaths:
err('No targetcov detailed per-gene report was generated; skipping.')
return None
open_tsv_files = [open(fpath) for fpath in fpaths]
first_col = 0
while True:
lines_for_each_sample = [next(f, None) for f in open_tsv_files]
if not all(lines_for_each_sample):
break
l = lines_for_each_sample[0]
if l.startswith('##'):
continue
elif l.startswith('#'):
if l.startswith('#Sample'):
first_col = 1
break
while True:
lines_for_each_sample = [next(f, None) for f in open_tsv_files]
if not all(lines_for_each_sample):
break
if all([
not l.startswith('#')
and ('Whole-Gene' in l or 'Gene-Exon' in l)
for l in lines_for_each_sample
]):
shared_fields = lines_for_each_sample[0].split(
'\t')[first_col:first_col + 9]
reg = report.add_row()
reg.add_record('Chr', get_val(shared_fields[0]))
reg.add_record('Start', get_int_val(shared_fields[1]))
reg.add_record('End', get_int_val(shared_fields[2]))
reg.add_record('Size', get_int_val(shared_fields[3]))
reg.add_record('Gene', get_val(shared_fields[4]))
reg.add_record('Strand', get_val(shared_fields[5]))
reg.add_record('Feature', get_val(shared_fields[6]))
reg.add_record('Biotype', get_val(shared_fields[7]))
reg.add_record('Transcript', get_val(shared_fields[8]))
min_depths, ave_depths, stddevs, withins = ([], [], [], [])
percents_by_threshs = {t: [] for t in depth_threshs}
for l in lines_for_each_sample:
fs = l.split('\t')
min_depths.append(get_int_val(fs[first_col + 9]))
ave_depths.append(get_float_val(fs[first_col + 10]))
stddevs.append(get_float_val(fs[first_col + 11]))
withins.append(get_float_val(fs[first_col + 12]))
for t, f in zip(depth_threshs, fs[first_col + 13:]):
percents_by_threshs[t].append(get_float_val(f))
# counting bests
reg.add_record('Min depth', select_best(min_depths))
reg.add_record('Ave depth', select_best(ave_depths))
reg.add_record('Std dev', select_best(stddevs, max))
reg.add_record('W/n 20% of median depth', select_best(withins))
for t in depth_threshs:
reg.add_record('{}x'.format(t),
select_best(percents_by_threshs[t]))
total_regions += 1
for f in open_tsv_files:
f.close()
gene_report_basename = add_suffix(samples[0].targqc_region_tsv, 'best')
txt_rep_fpath = report.save_txt(
join(output_dir, gene_report_basename + '.txt'))
tsv_rep_fpath = report.save_tsv(
join(output_dir, gene_report_basename + '.tsv'))
info('')
info('Best values for the regions (total ' + str(total_regions) +
') saved into:')
info(' ' + txt_rep_fpath)
return txt_rep_fpath
def proc_fastq(samples, parall_view, work_dir, bwa_prefix, downsample_to, num_pairs_by_sample=None, dedup=True):
num_pairs_by_sample = num_pairs_by_sample or dict()
if downsample_to:
# Read pairs counts
debug()
if all(s.name in num_pairs_by_sample for s in samples):
debug('Using read pairs counts extracted from FastQC reports')
elif all(can_reuse(make_pair_counts_fpath(join(work_dir, s.name)), s.l_fpath) for s in samples):
debug('Reusing pairs counts, reading from files')
num_pairs_by_sample = {s.name: int(open(make_pair_counts_fpath(join(work_dir, s.name))).read().strip()) for s in samples}
else:
info('Counting read pairs')
num_pairs = parall_view.run(count_read_pairs, [[s.name, safe_mkdir(join(work_dir, s.name)), s.l_fpath] for s in samples])
num_pairs_by_sample = {s.name: pairs_count for s, pairs_count in zip(samples, num_pairs)}
# Downsampling
debug()
if all(can_reuse(make_downsampled_fpath(join(work_dir, s.name), s.l_fpath), s.l_fpath) and
can_reuse(make_downsampled_fpath(join(work_dir, s.name), s.r_fpath), s.r_fpath) for s in samples):
debug('Reusing downsampled FastQ')
for s in samples:
s.l_fpath = make_downsampled_fpath(join(work_dir, s.name), s.l_fpath)
s.r_fpath = make_downsampled_fpath(join(work_dir, s.name), s.r_fpath)
else:
if isinstance(downsample_to, float):
info('Downsampling FastQ to ' + str(float(downsample_to)) + ' fraction of reads')
else:
info('Downsampling FastQ to ' + str(int(downsample_to)) + ' read pairs')
fastq_pairs = parall_view.run(downsample,
[[join(work_dir, s.name), s.name, s.l_fpath, s.r_fpath, downsample_to, num_pairs_by_sample.get(s.name)]
for s in samples])
for s, (l_r, r_r) in zip(samples, fastq_pairs):
s.l_fpath = l_r
s.r_fpath = r_r
else:
info('Skipping downsampling')
debug()
if all(can_reuse(make_bam_fpath(join(work_dir, s.name)), [s.l_fpath, s.r_fpath]) for s in samples):
debug('All downsampled BAM exists, reusing')
for s in samples:
s.bam = make_bam_fpath(join(work_dir, s.name))
else:
bwa = which('bwa')
if not isfile(bwa):
critical('BWA not found under ' + bwa)
smb = sambamba.get_executable()
if not (bwa and smb):
if not bwa: err('Error: bwa is required for the alignment pipeline')
if not smb: err('Error: sambamba is required for the alignment pipeline')
critical('Tools required for alignment not found')
info('Aligning reads to the reference')
bam_fpaths = parall_view.run(align,
[[join(work_dir, s.name), s.name, s.l_fpath, s.r_fpath, bwa, smb, bwa_prefix, dedup, parall_view.cores_per_job]
for s in samples])
bam_fpaths = [verify_bam(b) for b in bam_fpaths]
if len(bam_fpaths) < len(samples):
critical('Some samples were not aligned successfully.')
for bam, s in zip(bam_fpaths, samples):
s.bam = bam
return num_pairs_by_sample
def _save_best_details_for_each_gene(depth_threshs, samples, output_dir):
metric_storage = get_detailed_metric_storage(depth_threshs)
report = PerRegionSampleReport(sample='Best', metric_storage=metric_storage)
report.add_record('Sample', 'contains best values from all samples: ' + ', '.join([s.name for s in samples]))
total_regions = 0
fpaths = [s.targqc_region_tsv for s in samples if verify_file(s.targqc_region_tsv)]
if not fpaths:
err('No targetcov detailed per-gene report was generated; skipping.')
return None
open_tsv_files = [open(fpath) for fpath in fpaths]
first_col = 0
while True:
lines_for_each_sample = [next(f, None) for f in open_tsv_files]
if not all(lines_for_each_sample):
break
l = lines_for_each_sample[0]
if l.startswith('##'):
continue
elif l.startswith('#'):
if l.startswith('#Sample'):
first_col = 1
break
while True:
lines_for_each_sample = [next(f, None) for f in open_tsv_files]
if not all(lines_for_each_sample):
break
if all([not l.startswith('#') and ('Whole-Gene' in l or 'Gene-Exon' in l) for l in lines_for_each_sample]):
shared_fields = lines_for_each_sample[0].split('\t')[first_col:first_col+9]
reg = report.add_row()
reg.add_record('Chr', get_val(shared_fields[0]))
reg.add_record('Start', get_int_val(shared_fields[1]))
reg.add_record('End', get_int_val(shared_fields[2]))
reg.add_record('Size', get_int_val(shared_fields[3]))
reg.add_record('Gene', get_val(shared_fields[4]))
reg.add_record('Strand', get_val(shared_fields[5]))
reg.add_record('Feature', get_val(shared_fields[6]))
reg.add_record('Biotype', get_val(shared_fields[7]))
reg.add_record('Transcript', get_val(shared_fields[8]))
min_depths, ave_depths, stddevs, withins = ([], [], [], [])
percents_by_threshs = {t: [] for t in depth_threshs}
for l in lines_for_each_sample:
fs = l.split('\t')
min_depths.append(get_int_val(fs[first_col+9]))
ave_depths.append(get_float_val(fs[first_col+10]))
stddevs.append(get_float_val(fs[first_col+11]))
withins.append(get_float_val(fs[first_col+12]))
for t, f in zip(depth_threshs, fs[first_col+13:]):
percents_by_threshs[t].append(get_float_val(f))
# counting bests
reg.add_record('Min depth', select_best(min_depths))
reg.add_record('Ave depth', select_best(ave_depths))
reg.add_record('Std dev', select_best(stddevs, max))
reg.add_record('W/n 20% of median depth', select_best(withins))
for t in depth_threshs:
reg.add_record('{}x'.format(t), select_best(percents_by_threshs[t]))
total_regions += 1
for f in open_tsv_files:
f.close()
gene_report_basename = add_suffix(samples[0].targqc_region_tsv, 'best')
txt_rep_fpath = report.save_txt(join(output_dir, gene_report_basename + '.txt'))
tsv_rep_fpath = report.save_tsv(join(output_dir, gene_report_basename + '.tsv'))
info('')
info('Best values for the regions (total ' + str(total_regions) + ') saved into:')
info(' ' + txt_rep_fpath)
return txt_rep_fpath