def get_vcf_readers(mutations_by_experiment, cur_group_num):
vcf_readers, filt_vcf_readers = dict(), dict()
for e, muts in mutations_by_experiment.items():
if not cur_group_num or get_group_num(e.key) == cur_group_num:
variant_caller = 'vardict' if 'vardict' in e.sample.variantcallers else 'vardict-java'
if e.sample.vcf_by_callername.get(variant_caller):
vcf_fpath = e.sample.vcf_by_callername.get(variant_caller)
filt_vcf_fpath = e.sample.find_filt_vcf_by_callername(
variant_caller)
if vcf_fpath:
vcf_readers[e] = vcf.Reader(open_gzipsafe(vcf_fpath, 'r'))
if filt_vcf_fpath:
filt_vcf_readers[e] = vcf.Reader(
open_gzipsafe(filt_vcf_fpath, 'r'))
return vcf_readers, filt_vcf_readers
def verify_vcf(vcf_fpath, silent=False, is_critical=False):
if not verify_file(vcf_fpath, silent=silent, is_critical=is_critical):
return None
debug('File ' + vcf_fpath + ' exists and not empty')
vcf = open_gzipsafe(vcf_fpath)
debug('File ' + vcf_fpath + ' opened')
l = next(vcf, None)
if l is None:
(critical if is_critical else err)('Error: cannot read the VCF file ' + vcf_fpath)
return None
if not l.startswith('##fileformat=VCF'):
(critical if is_critical else err)('Error: VCF must start with ##fileformat=VCF ' + vcf_fpath)
return None
try:
reader = vcf_parser.Reader(vcf)
except:
err('Error: cannot open the VCF file ' + vcf_fpath)
if is_critical: raise
else:
debug('File ' + vcf_fpath + ' opened as VCF')
try:
rec = next(reader)
except IndexError:
err('Error: cannot parse records in the VCF file ' + vcf_fpath)
debug('IndexError parsing VCF file ' + vcf_fpath)
if is_critical: raise
except ValueError:
err('Error: cannot parse records in the VCF file ' + vcf_fpath)
debug('ValueError parsing VCF file ' + vcf_fpath)
if is_critical: raise
except StopIteration:
debug('No records in the VCF file ' + vcf_fpath)
if not silent:
warn('VCF file ' + vcf_fpath + ' has no records.')
return vcf_fpath
except:
err('Error: cannot parse records in the VCF file ' + vcf_fpath)
debug('Other error parsing VCF file ' + vcf_fpath)
if is_critical: raise
else:
debug('A record was read from the VCF file ' + vcf_fpath)
return vcf_fpath
# f = open_gzipsafe(output_fpath)
# l = f.readline()
# if 'Cannot allocate memory' in l:
# f.close()
# f = open_gzipsafe(output_fpath)
# contents = f.read()
# if not silent:
# if is_critical:
# critical('SnpSift failed with memory issue:\n' + contents)
# else:
# err('SnpSift failed with memory issue:\n' + contents)
# return None
# f.close()
# return None
# return output_fpath
finally:
vcf.close()
def _convert_vcf(inp_f, out_f):
max_bunch_size = 100000
written_records = 0
bunch = []
reader = vcf_parser.Reader(inp_f)
writer = vcf_parser.Writer(out_f, reader)
i = 0
while True:
rec = next(reader, None)
if rec is None:
break
rec = proc_rec_fun(Record(rec, input_fpath, i), *args, **kwargs)
if rec:
bunch.append(rec)
written_records += 1
if len(bunch) >= max_bunch_size:
writer.write_records(bunch)
info('Written lines: ' + str(written_records))
bunch = []
i += 1
writer.write_records(bunch)
bunch = []
info('Written lines: ' + str(written_records))
def vcf_is_empty(cnf, vcf_fpath):
vcf = open_gzipsafe(vcf_fpath)
reader = vcf_parser.Reader(vcf)
result = True
for rec in reader:
result = False
vcf.close()
return result
def _get_subs_and_indel_stats(vcf_fpath, chr_lengths, plot_scale):
reader = vcf.Reader(open_gzipsafe(vcf_fpath, 'r'))
variants_distribution = dict()
for chr_name, chr_length in chr_lengths:
variants_distribution[chr_name] = [0] * max(1, chr_length / plot_scale)
variants_distribution['OTHER'] = 0
substituitions = OrderedDict()
nucleotides = ['A', 'C', 'G', 'T']
def _add_nuc(nuc):
substituitions[nuc] = OrderedDict()
for nuc2 in nucleotides:
if nuc != nuc2:
substituitions[nuc][nuc2] = 0
for nuc in nucleotides:
_add_nuc(nuc)
indel_lengths = []
for rec in reader:
# for variants distribution plot
if rec.CHROM not in variants_distribution:
variants_distribution['OTHER'] += 1
else:
region_id = min((rec.POS - 1) / plot_scale, len(variants_distribution[rec.CHROM]) - 1)
variants_distribution[rec.CHROM][region_id] += 1
# for substitution and indel plots
for alt in rec.ALT:
if rec.is_snp:
if rec.REF not in substituitions:
nucleotides.append(rec.REF)
_add_nuc(rec.REF)
if alt.sequence not in substituitions:
nucleotides.append(alt.sequence)
_add_nuc(alt.sequence)
substituitions[rec.REF][str(alt)] += 1
elif rec.is_indel:
if alt is None:
indel_lengths.append(-1)
else:
indel_lengths.append(len(alt) - len(rec.REF))
# the last region in each chromosome is not exactly equal to plot_scale
for chr_name, chr_length in chr_lengths:
last_region_length = chr_length % plot_scale + (0 if chr_length < plot_scale else plot_scale)
variants_distribution[chr_name][-1] = int(variants_distribution[chr_name][-1] * plot_scale /
float(last_region_length))
return variants_distribution, substituitions, indel_lengths
def make_report(cnf, vcf_fpath, sample):
set_db_versions(cnf)
step_greetings('Quality control reports')
total_with_rejected = 0
total = 0
snps = 0
inss = 0
dels = 0
dbsnps = 0
cosmics = 0
novels = 0
hets = 0
homs = 0
transitions = 0
transversions = 0
with open_gzipsafe(vcf_fpath) as f:
reader = vcf_parser.Reader(f)
for rec in (vcf_processing.Record(rec, vcf_fpath, i)
for i, rec in enumerate(reader)):
total_with_rejected += 1
if not rec.FILTER or rec.FILTER == 'PASS':
if rec.FILTER:
warn('Warn: ' + rec.get_variant() + ' FILTER=' +
str(rec.FILTER))
total += 1
if rec.is_snp:
snps += 1
if rec.is_transition:
transitions += 1
elif len(rec.ALT) == 1:
transversions += 1
elif rec.is_indel:
if rec.is_deletion:
dels += 1
elif len(rec.ALT) == 1:
inss += 1
if not rec.ID:
novels += 1
else:
ids = rec.ID
if isinstance(ids, basestring):
ids = [ids]
if any(id.startswith('COS') for id in ids):
cosmics += 1
if any(id.startswith('rs') for id in ids):
dbsnps += 1
call = rec.samples[0]
if call.called:
if call.gt_type == 1:
hets += 1
elif call.gt_type == 2:
homs += 1
report = SampleReport(sample, metric_storage=metric_storage)
report.add_record('Total variants', total)
report.add_record('SNPs', snps)
report.add_record('Insertions', inss)
report.add_record('Deletions', dels)
report.add_record('Novel', novels)
report.add_record('Novel, %', 1.0 * novels / total if total else None)
report.add_record('In dbSNP', dbsnps)
report.add_record('In dbSNP, %', 1.0 * dbsnps / total if total else None)
report.add_record('In Cosmic', cosmics)
report.add_record('In Cosmic, %', 1.0 * cosmics / total if total else None)
report.add_record('Het/hom', float(hets) / homs if homs != 0 else None)
report.add_record(
'Ti/tv',
float(transitions) / transversions if transversions != 0 else None)
report.add_record('Total with rejected', total_with_rejected)
return report
def _extract_fields(cnf, vcf_fpath, samplename, main_sample_index=0):
fname, _ = splitext_plus(basename(vcf_fpath))
tsv_fpath = join(cnf.work_dir, fname + '.tsv')
if cnf.get('reuse_intermediate'):
if file_exists(tsv_fpath):
info(tsv_fpath + ' exists, reusing')
return tsv_fpath
manual_tsv_fields = cnf.annotation['tsv_fields']
if not manual_tsv_fields:
return None
all_fields = []
basic_fields = []
info_fields = []
eff_fields = []
gt_fields = []
tumor_gt = 'GEN[' + str(main_sample_index) + '].'
normal_gt = 'GEN[' + str(1 - main_sample_index) + '].'
lines = []
with open(vcf_fpath) as inp:
reader = vcf.Reader(inp)
info('TSV saver: Building field list')
for f in [rec.keys()[0] for rec in manual_tsv_fields]:
if f.startswith('GEN'):
_f = f.split('.')[1]
if len(reader.samples) > 0:
if _f in reader.formats:
gt_fields.append(_f)
all_fields.append(f.replace('GEN[*].', tumor_gt))
if len(reader.samples) > 1:
all_fields.append(f.replace('GEN[*].', normal_gt))
else:
warn('TSV Saver: Warning: ' + f + ' is not in VCF header FORMAT records')
elif f in ['CHROM', 'POS', 'REF', 'ALT', 'ID', 'FILTER', 'QUAL']:
all_fields.append(f)
basic_fields.append(f)
elif any(f.startswith(af) and af in reader.infos for af in ['EFF', 'ANN']):
all_fields.append(f)
eff_fields.append(f)
else:
if f in reader.infos:
info_fields.append(f)
all_fields.append(f)
elif f == 'SAMPLE':
all_fields.append(f)
else:
warn('TSV Saver: Warning: ' + f + ' is not in VCF header INFO records')
info('TSV saver: Iterating over records...')
d = OrderedDict()
for rec in reader:
for f in basic_fields:
d[f] = rec.__dict__[f]
for f in info_fields:
d[f] = rec.INFO[f] if f in rec.INFO else ''
if 'SAMPLE' not in d:
d['SAMPLE'] = samplename
if eff_fields:
eff = rec.INFO.get(eff_fields[0][:3])
if not eff:
for f in eff_fields:
d[f] = ''
else:
eff_fs = eff[0].split('|')
eff_d = dict()
for val, header in zip(eff_fs, ['ALLELE', 'EFFECT', 'IMPACT', 'GENE', 'GENEID', 'FEATURE', 'FEATUREID', 'BIOTYPE', 'RANK', 'HGVS_C', 'HGVS_P', 'CDNA_POSLEN', 'CDS_POSLEN', 'AA_POSLEN', 'DISTANCE', 'LOG']):
if 'POSLEN' in header:
eff_d[header.split('_')[0] + '_POS'] = val.split('/')[0] if val else ''
eff_d[header.split('_')[0] + '_LEN'] = val.split('/')[1] if val else ''
else:
eff_d[header] = val
#ANN=GA |3_prime_UTR_variant|MODIFIER|RPL22|RPL22|transcript|NM_000983.3|Coding|4/4|c.*173dupT|||||173|;
#Allele | Annotation | Annotation_Impact | Gene_Name | Gene_ID | Feature_Type | Feature_ID | Transcript_BioType | Rank | HGVS.c | HGVS.p | cDNA.pos / cDNA.length | CDS.pos / CDS.length | AA.pos / AA.length | Distance | ERRORS / WARNINGS / INFO'
for f in eff_fields:
d[f] = eff_d[f.split('.')[1]]
if rec.FORMAT:
for _f in gt_fields:
if _f in rec.FORMAT:
d[tumor_gt + _f] = rec.samples[main_sample_index][_f]
if len(rec.samples) > 1 - main_sample_index:
d[normal_gt + _f] = rec.samples[1 - main_sample_index][_f]
else:
d[normal_gt + _f] = ''
else:
d[tumor_gt + _f] = ''
d[normal_gt + _f] = ''
fs = []
for f in all_fields:
v = d[f]
fs.append(v if v != '.' else '')
lines.append(fs)
info('TSV saver: Adding GEN[*] fields both for sample and for matched normal...')
field_map = dict()
for rec in manual_tsv_fields:
k = rec.keys()[0]
v = rec.values()[0]
if k.startswith('GEN[*].'):
_f = k.split('.')[1]
field_map[tumor_gt + _f] = v
field_map[normal_gt + _f] = 'Matched_' + v
else:
field_map[k] = v
info('TSV saver: Writing TSV to ' + tsv_fpath)
with file_transaction(cnf.work_dir, tsv_fpath) as tx:
with open(tx, 'w') as out:
out.write('\t'.join(field_map[f] for f in all_fields) + '\n')
for fs in lines:
new_fs = []
for f in fs:
if isinstance(f, list):
new_fs.append(','.join(map(str, f)))
elif f is None:
new_fs.append('')
else:
new_fs.append(str(f))
out.write('\t'.join(new_fs) + '\n')
info('TSV saver: saved ' + tsv_fpath)
return tsv_fpath
def convert_vardict_txts_to_bcbio_vcfs(cnf,
bs,
sample,
output_dir=None,
pass_only=False):
info('')
info('Preparing data for ' + sample.name)
anno_filt_vcf_fpath = sample.find_filt_vcf_by_callername(cnf.caller_name)
if not anno_filt_vcf_fpath:
return None, None
if not output_dir:
output_dir = cnf.output_dir or os.path.dirname(anno_filt_vcf_fpath)
output_vcf_fpath = join(
output_dir, sample.name + '-' + cnf.caller_name + filt_vcf_ending)
pass_output_vcf_fpath = add_suffix(output_vcf_fpath, 'pass')
if cnf.reuse_intermediate and verify_vcf(
output_vcf_fpath + '.gz') and verify_vcf(pass_output_vcf_fpath +
'.gz'):
info(output_vcf_fpath + '.gz and ' + pass_output_vcf_fpath +
'.gz exists, reusing')
return output_vcf_fpath + '.gz', pass_output_vcf_fpath + '.gz'
info('Parsing PASS and REJECT mutations...')
pass_mut_dict, reject_mut_dict, filter_values = get_mutation_dicts(
cnf, bs, sample, pass_only=pass_only)
sorted_mut_dict = combine_mutations(pass_mut_dict, reject_mut_dict)
info('')
info('Writing VCFs')
vcf_reader = vcf.Reader(open_gzipsafe(anno_filt_vcf_fpath, 'r'))
vcf_reader = add_keys_to_header(vcf_reader, filter_values)
with file_transaction(cnf.work_dir, output_vcf_fpath) as filt_tx, \
file_transaction(cnf.work_dir, pass_output_vcf_fpath) as pass_tx:
vcf_writer = None
if not pass_only:
vcf_writer = vcf.Writer(open(filt_tx, 'w'), template=vcf_reader)
vcf_pass_writer = vcf.Writer(open(pass_tx, 'w'), template=vcf_reader)
for key, mut in sorted_mut_dict.items():
record = get_record_from_vcf(vcf_reader, mut)
if record:
if key in pass_mut_dict:
record.FILTER = ['PASS']
if mut.reason:
record.INFO['Reason'] = mut.reason.replace(' ', '_')
elif pass_only:
continue
elif key in reject_mut_dict:
if not mut.reason:
continue
reject_reason_ids = [
filter_descriptions_dict[reason]
if reason in filter_descriptions_dict else reason
for reason in mut.reason.split(' and ')
]
record.FILTER = [';'.join(reject_reason_ids)]
if mut.signif:
record.INFO['Signif'] = mut.signif
if mut.status:
record.INFO['Status'] = mut.status
if vcf_writer:
vcf_writer.write_record(record)
if key in pass_mut_dict:
vcf_pass_writer.write_record(record)
else:
warn('No record was found in ' + anno_filt_vcf_fpath +
' for mutation ' + str(mut))
output_gzipped_vcf_fpath = None
if vcf_writer:
vcf_writer.close()
output_gzipped_vcf_fpath = bgzip_and_tabix(cnf, output_vcf_fpath)
info('VCF file for vardict.txt is saved to ' +
output_gzipped_vcf_fpath)
vcf_pass_writer.close()
output_gzipped_pass_vcf_fpath = bgzip_and_tabix(cnf, pass_output_vcf_fpath)
info('VCF file for vardict.PASS.txt is saved to ' +
output_gzipped_pass_vcf_fpath)
return output_gzipped_vcf_fpath, output_gzipped_pass_vcf_fpath