def check_contig_in_bam(bam_fn, sorted_contig_list, samtools):
bai_process = subprocess_popen(shlex.split("{} idxstats {}".format(samtools, bam_fn)))
contig_with_read_support_set = set()
for row_id, row in enumerate(bai_process.stdout):
row = row.split('\t')
if len(row) != 4:
continue
contig_name, contig_length, mapped_reads, unmapped_reads = row
if contig_name not in sorted_contig_list:
continue
if int(mapped_reads) > 0:
contig_with_read_support_set.add(contig_name)
for contig_name in sorted_contig_list:
if contig_name not in contig_with_read_support_set:
print(log_warning(
"[WARNING] Contig name {} provided but no mapped reads in BAM, skip!".format(contig_name)))
filtered_sorted_contig_list = [item for item in sorted_contig_list if item in contig_with_read_support_set]
found_contig = True
if len(filtered_sorted_contig_list) == 0:
found_contig = False
print(log_warning(
"[WARNING] No mapped reads support in BAM for provided contigs set {}".format(
' '.join(sorted_contig_list))))
return filtered_sorted_contig_list, found_contig
def reference_sequence_from(samtools_execute_command, fasta_file_path,
regions):
refernce_sequences = []
region_value_for_faidx = " ".join(regions)
samtools_faidx_process = subprocess_popen(
shlex.split("{} faidx {} {}".format(samtools_execute_command,
fasta_file_path,
region_value_for_faidx)))
while True:
row = samtools_faidx_process.stdout.readline()
is_finish_reading_output = row == '' and samtools_faidx_process.poll(
) is not None
if is_finish_reading_output:
break
if row:
refernce_sequences.append(row.rstrip())
# first line is reference name ">xxxx", need to be ignored
reference_sequence = "".join(refernce_sequences[1:])
# uppercase for masked sequences
reference_sequence = reference_sequence.upper()
samtools_faidx_process.stdout.close()
samtools_faidx_process.wait()
if samtools_faidx_process.returncode != 0:
return None
return reference_sequence
def bed_tree_from(bed_file_path):
"""
0-based interval tree [start, end)
"""
tree = {}
if bed_file_path is None:
return tree
unzip_process = subprocess_popen(
shlex.split("gzip -fdc %s" % (bed_file_path)))
while True:
row = unzip_process.stdout.readline()
is_finish_reading_output = row == '' and unzip_process.poll(
) is not None
if is_finish_reading_output:
break
if row:
columns = row.strip().split()
ctg_name = columns[0]
if ctg_name not in tree:
tree[ctg_name] = IntervalTree()
ctg_start, ctg_end = int(columns[1]), int(columns[2])
if ctg_start == ctg_end:
ctg_end += 1
tree[ctg_name].addi(ctg_start, ctg_end)
unzip_process.stdout.close()
unzip_process.wait()
return tree
def variants_map_from(variant_file_path):
"""
variants map with 1-based position as key
"""
if variant_file_path == None:
return {}
variants_map = {}
f = subprocess_popen(shlex.split("gzip -fdc %s" % (variant_file_path)))
while True:
row = f.stdout.readline()
is_finish_reading_output = row == '' and f.poll() is not None
if is_finish_reading_output:
break
if row:
columns = row.split()
ctg_name, position_str = columns[0], columns[1]
key = ctg_name + ":" + position_str
variants_map[key] = True
f.stdout.close()
f.wait()
return variants_map
def split_extend_vcf(vcf_fn, output_fn):
expand_region_size = param.no_of_positions
output_ctg_dict = defaultdict(list)
unzip_process = subprocess_popen(shlex.split("gzip -fdc %s" % (vcf_fn)))
for row_id, row in enumerate(unzip_process.stdout):
if row[0] == '#':
continue
columns = row.strip().split(maxsplit=3)
ctg_name = columns[0]
center_pos = int(columns[1])
ctg_start, ctg_end = center_pos - 1, center_pos
if ctg_start < 0:
sys.exit(
log_error("[ERROR] Invalid VCF input in {}-th row {} {} {}".format(row_id + 1, ctg_name, center_pos)))
if ctg_start - expand_region_size < 0:
continue
expand_ctg_start = ctg_start - expand_region_size
expand_ctg_end = ctg_end + expand_region_size
output_ctg_dict[ctg_name].append(
' '.join([ctg_name, str(expand_ctg_start), str(expand_ctg_end)]))
for key, value in output_ctg_dict.items():
ctg_output_fn = os.path.join(output_fn, key)
with open(ctg_output_fn, 'w') as output_file:
output_file.write('\n'.join(value))
unzip_process.stdout.close()
unzip_process.wait()
know_vcf_contig_set = set(list(output_ctg_dict.keys()))
return know_vcf_contig_set
def split_extend_bed(bed_fn, output_fn, contig_set=None):
expand_region_size = param.no_of_positions
output_ctg_dict = defaultdict(list)
unzip_process = subprocess_popen(shlex.split("gzip -fdc %s" % (bed_fn)))
for row_id, row in enumerate(unzip_process.stdout):
if row[0] == '#':
continue
columns = row.strip().split()
ctg_name = columns[0]
if contig_set and ctg_name not in contig_set:
continue
ctg_start, ctg_end = int(columns[1]), int(columns[2])
if ctg_end < ctg_start or ctg_start < 0 or ctg_end < 0:
sys.exit(log_error(
"[ERROR] Invalid BED input in {}-th row {} {} {}".format(row_id + 1, ctg_name, ctg_start, ctg_end)))
expand_ctg_start = max(0, ctg_start - expand_region_size)
expand_ctg_end = max(0, ctg_end + expand_region_size)
output_ctg_dict[ctg_name].append(
' '.join([ctg_name, str(expand_ctg_start), str(expand_ctg_end)]))
for key, value in output_ctg_dict.items():
ctg_output_fn = os.path.join(output_fn, key)
with open(ctg_output_fn, 'w') as output_file:
output_file.write('\n'.join(value))
unzip_process.stdout.close()
unzip_process.wait()
def variant_map_from(var_fn, tree, is_tree_empty):
Y = {}
truth_alt_dict = {}
miss_variant_set = set()
if var_fn is None:
return Y, miss_variant_set, truth_alt_dict
f = subprocess_popen(shlex.split("gzip -fdc %s" % (var_fn)))
for row in f.stdout:
if row[0] == "#":
continue
columns = row.strip().split()
ctg_name, position_str, ref_base, alt_base, genotype1, genotype2 = columns
key = ctg_name + ":" + position_str
if genotype1 == '-1' or genotype2 == '-1':
miss_variant_set.add(key)
continue
if not (is_tree_empty or is_region_in(tree, ctg_name, int(position_str))):
continue
Y[key] = output_labels_from_vcf_columns(columns)
ref_base_list, alt_base_list = decode_alt(ref_base, alt_base)
truth_alt_dict[int(position_str)] = (ref_base_list, alt_base_list)
f.stdout.close()
f.wait()
return Y, miss_variant_set, truth_alt_dict
def read_input(self):
if self.compress:
self.read_proc = subprocess_popen(shlex.split("{} {}".format(
LZ4_DECOMPRESS, self.input_path)),
stderr=subprocess.DEVNULL)
self.reader = self.read_proc.stdout
else:
self.reader = open(self.input_path, 'r')
return self.reader
def FiterHeteSnpPhasing(args):
"""
Filter heterozygous snp variant for phasing, currently, we only filter snp variant with low quality socore as low
quality variant contains more false positive variant that would lead to a larger minimum error correction loss.
"""
qual_fn = args.qual_fn if args.qual_fn is not None else 'phase_qual'
vcf_fn = args.vcf_fn
var_pct_full = args.var_pct_full
contig_name = args.ctgName
split_folder = args.split_folder
variant_dict = defaultdict(str)
qual_set = defaultdict(int)
found_qual_cut_off = False
header = []
#try to find the global quality cut off:
f_qual = os.path.join(split_folder, qual_fn)
if os.path.exists(f_qual):
phase_qual_cut_off = float(open(f_qual, 'r').read().rstrip())
found_qual_cut_off = True
unzip_process = subprocess_popen(shlex.split("gzip -fdc %s" % (vcf_fn)))
for row in unzip_process.stdout:
row = row.rstrip()
if row[0] == '#':
header.append(row + '\n')
continue
columns = row.strip().split()
ctg_name = columns[0]
if contig_name and contig_name != ctg_name:
continue
pos = int(columns[1])
ref_base = columns[3]
alt_base = columns[4]
genotype = columns[9].split(':')[0].replace('|', '/')
if len(ref_base) == 1 and len(alt_base) == 1:
if genotype == '0/1' or genotype=='1/0':
variant_dict[pos] = row
qual = float(columns[5])
qual_set[pos] = qual
if found_qual_cut_off:
remove_low_qual_list = [[k,v] for k,v in qual_set.items() if v < phase_qual_cut_off ]
else:
remove_low_qual_list = sorted(qual_set.items(), key=lambda x: x[1])[:int(var_pct_full * len(qual_set))]
for pos, qual in remove_low_qual_list:
del variant_dict[pos]
print ('[INFO] Total heterozygous SNP positions selected: {}: {}'.format(contig_name, len(variant_dict)))
f = open(os.path.join(split_folder, '{}.vcf'.format(contig_name)), 'w')
f.write(''.join(header))
for key,row in sorted(variant_dict.items(), key=lambda x: x[0]):
f.write(row +'\n')
f.close()
def GetBase(chromosome, position, ref_fn):
fp = subprocess_popen(
shlex.split("samtools faidx %s %s:%s-%s" %
(ref_fn, chromosome, position, position)))
for line in fp.stdout:
if line[0] == ">":
continue
else:
return line.strip()
def samtools_view_process_from(ctg_name, ctg_start, ctg_end, samtools,
bam_file_path):
have_start_and_end_position = ctg_start != None and ctg_end != None
region_str = ("%s:%d-%d" % (ctg_name, ctg_start, ctg_end)
) if have_start_and_end_position else ctg_name
return subprocess_popen(
shlex.split("%s view -F %d %s %s" %
(samtools, param.SAMTOOLS_VIEW_FILTER_FLAG, bam_file_path,
region_str)))
def write_output(self):
if self.compress:
self.write_fpo = open(self.output_path, 'w')
self.write_proc = subprocess_popen(shlex.split(LZ4_COMPRESS),
stdin=subprocess.PIPE,
stdout=self.write_fpo,
stderr=subprocess.DEVNULL)
self.writer = self.write_proc.stdin
else:
self.writer = open(self.output_path, 'w')
return self.writer
def split_extend_bed(args):
"""
Split bed file regions according to the contig name and extend bed region with no_of_positions =
flankingBaseNum + 1 + flankingBaseNum, which allow samtools mpileup submodule to scan the flanking windows.
"""
bed_fn = args.bed_fn
output_fn = args.output_fn
contig_name = args.ctgName
region_start = args.ctgStart
region_end = args.ctgEnd
expand_region_size = args.expand_region_size
if bed_fn is None:
return
output = []
unzip_process = subprocess_popen(shlex.split("gzip -fdc %s" % (bed_fn)))
pre_end, pre_start = -1, -1
for row in unzip_process.stdout:
if row[0] == '#':
continue
columns = row.strip().split()
ctg_name = columns[0]
if contig_name != None and ctg_name != contig_name:
continue
ctg_start, ctg_end = int(columns[1]), int(columns[2])
if region_start and ctg_end < region_start:
continue
if region_end and ctg_start > region_end:
break
if pre_start == -1:
pre_start = ctg_start - expand_region_size
pre_end = ctg_end + expand_region_size
continue
if pre_end >= ctg_start - expand_region_size:
pre_end = ctg_end + expand_region_size
continue
else:
output.append(' '.join([contig_name, str(pre_start), str(pre_end)]))
pre_start = ctg_start - expand_region_size
pre_end = ctg_end + expand_region_size
with open(output_fn, 'w') as output_file:
output_file.write('\n'.join(output))
unzip_process.stdout.close()
unzip_process.wait()
def bed_tree_from(bed_file_path, expand_region=None, contig_name=None, bed_ctg_start=None, bed_ctg_end=None,
return_bed_region=False, padding=None):
"""
0-based interval tree [start, end)
"""
tree = {}
if bed_file_path is None:
if return_bed_region:
return tree, None, None
return tree
bed_start, bed_end = float('inf'), 0
unzip_process = subprocess_popen(shlex.split("gzip -fdc %s" % (bed_file_path)))
for row_id, row in enumerate(unzip_process.stdout):
if row[0] == '#':
continue
columns = row.strip().split()
ctg_name = columns[0]
if contig_name != None and ctg_name != contig_name:
continue
if ctg_name not in tree:
tree[ctg_name] = IntervalTree()
ctg_start, ctg_end = int(columns[1]), int(columns[2])
if ctg_end < ctg_start or ctg_start < 0 or ctg_end < 0:
sys.exit("[ERROR] Invalid bed input in {}-th row {} {} {}".format(row_id+1, ctg_name, ctg_start, ctg_end))
if bed_ctg_start and bed_ctg_end:
if ctg_end < bed_ctg_start or ctg_start > bed_ctg_end:
continue
if padding:
ctg_start += padding
ctg_end -= padding
bed_start = min(ctg_start, bed_start)
bed_end = max(ctg_end, bed_end)
if ctg_start == ctg_end:
ctg_end += 1
tree[ctg_name].addi(ctg_start, ctg_end)
unzip_process.stdout.close()
unzip_process.wait()
if return_bed_region:
return tree, bed_start, bed_end
return tree
def reference_result_from(
ctg_name,
ctg_start,
ctg_end,
samtools,
reference_file_path,
expand_reference_region
):
region_str = ""
reference_start, reference_end = None, None
have_start_and_end_positions = ctg_start != None and ctg_end != None
if have_start_and_end_positions:
reference_start, reference_end = ctg_start - expand_reference_region, ctg_end + expand_reference_region
reference_start = 1 if reference_start < 1 else reference_start
region_str = "%s:%d-%d" % (ctg_name, reference_start, reference_end)
else:
region_str = ctg_name
faidx_process = subprocess_popen(shlex.split("%s faidx %s %s" % (samtools, reference_file_path, region_str)),)
if faidx_process is None:
return None
reference_name = None
reference_sequences = []
for row in faidx_process.stdout:
if reference_name is None:
reference_name = row.rstrip().lstrip(">") or ""
else:
reference_sequences.append(row.rstrip())
reference_sequence = "".join(reference_sequences)
# uppercase for masked sequences
reference_sequence = reference_sequence.upper()
faidx_process.stdout.close()
faidx_process.wait()
return ReferenceResult(
name=reference_name,
start=reference_start,
end=reference_end,
sequence=reference_sequence,
is_faidx_process_have_error=faidx_process.returncode != 0,
)
def variant_map_from(var_fn, tree, is_tree_empty):
Y = {}
if var_fn is None:
return Y
f = subprocess_popen(shlex.split("gzip -fdc %s" % (var_fn)))
for row in f.stdout:
columns = row.split()
ctg_name, position_str = columns[0], columns[1]
if not (is_tree_empty or is_region_in(tree, ctg_name, int(position_str))):
continue
key = ctg_name + ":" + position_str
Y[key] = output_labels_from_vcf_columns(columns)
f.stdout.close()
f.wait()
return Y
def tensor_generator_from(tensor_file_path, batch_size):
if tensor_file_path != "PIPE":
f = subprocess_popen(shlex.split("gzip -fdc %s" % (tensor_file_path)))
fo = f.stdout
else:
fo = sys.stdin
processed_tensors = 0
def item_from(row):
# print(row)
columns = row.split()
return columns[:-input_tensor_size], np.array(
columns[-input_tensor_size:], dtype=np.float32)
for batch in batches_from(fo, item_from=item_from, batch_size=batch_size):
# tmp_time = time()
tensors = np.empty((batch_size, input_tensor_size), dtype=np.float32)
non_tensor_infos = []
for non_tensor_info, tensor in batch:
_, _, sequence = non_tensor_info
if sequence[param.flankingBaseNum] not in BASE2NUM:
continue
tensors[len(non_tensor_infos)] = tensor
non_tensor_infos.append(non_tensor_info)
current_batch_size = len(non_tensor_infos)
X = np.reshape(tensors,
(batch_size, no_of_positions, matrix_row, matrix_num))
for i in range(1, matrix_num):
X[:current_batch_size, :, :, i] -= X[:current_batch_size, :, :, 0]
processed_tensors += current_batch_size
# print("Processed %d tensors takes %.4f s" % (processed_tensors, time() - tmp_time), file=sys.stderr)
print("Processed %d tensors" % processed_tensors, file=sys.stderr)
if current_batch_size <= 0:
continue
yield X[:current_batch_size], non_tensor_infos[:current_batch_size]
if tensor_file_path != "PIPE":
fo.close()
f.wait()
def candidate_position_generator_from(candidate_file_path, ctg_start, ctg_end,
is_consider_left_edge, flanking_base_num,
begin_to_end):
is_read_file_from_standard_input = candidate_file_path == "PIPE"
if is_read_file_from_standard_input:
candidate_file_path_output = sys.stdin
else:
candidate_file_path_process = subprocess_popen(
shlex.split("gzip -fdc %s" % (candidate_file_path)))
candidate_file_path_output = candidate_file_path_process.stdout
is_ctg_region_provided = ctg_start is not None and ctg_end is not None
for row in candidate_file_path_output:
row = row.split()
position = int(row[1]) # 1-based position
if is_ctg_region_provided and not (ctg_start <= position <= ctg_end):
continue
if is_consider_left_edge:
# i is 0-based
for i in range(position - (flanking_base_num + 1),
position + (flanking_base_num + 1)):
if i not in begin_to_end:
begin_to_end[i] = [(position + (flanking_base_num + 1),
position)]
else:
begin_to_end[i].append(
(position + (flanking_base_num + 1), position))
else:
begin_to_end[position - (flanking_base_num + 1)] = [
(position + (flanking_base_num + 1), position)
]
yield position
if not is_read_file_from_standard_input:
candidate_file_path_output.close()
candidate_file_path_process.wait()
yield -1
def Run(args):
tree = bed_tree_from(bed_file_path=args.bed_fn)
logging.info("Counting the number of Truth Variants in %s ..." % args.tensor_var_fn)
v = 0
d = {}
f = subprocess_popen(shlex.split("gzip -fdc %s" % (args.tensor_var_fn)))
for row in f.stdout:
row = row.strip().split()
ctgName = row[0]
pos = int(row[1])
key = "-".join([ctgName, str(pos)])
v += 1
d[key] = 1
f.stdout.close()
f.wait()
logging.info("%d Truth Variants" % v)
t = v * args.amp
logging.info("%d non-variants to be picked" % t)
logging.info("Counting the number of usable non-variants in %s ..." % args.tensor_can_fn)
c = 0
f = subprocess_popen(shlex.split("gzip -fdc %s" % (args.tensor_can_fn)))
for row in f.stdout:
row = row.strip().split()
ctgName = row[0]
pos = int(row[1])
if args.bed_fn != None:
if not is_region_in(tree, ctgName, pos):
continue
key = "-".join([ctgName, str(pos)])
if key in d:
continue
c += 1
f.stdout.close()
f.wait()
logging.info("%d usable non-variant" % c)
r = float(t) / c
r = r if r <= 1 else 1
logging.info("%.2f of all non-variants are selected" % r)
o1 = 0
o2 = 0
output_fpo = open(args.output_fn, "wb")
output_fh = subprocess_popen(shlex.split("gzip -c"), stdin=PIPE, stdout=output_fpo)
f = subprocess_popen(shlex.split("gzip -fdc %s" % (args.tensor_var_fn)))
for row in f.stdout:
row = row.strip()
output_fh.stdin.write(row)
output_fh.stdin.write("\n")
o1 += 1
f.stdout.close()
f.wait()
f = subprocess_popen(shlex.split("gzip -fdc %s" % (args.tensor_can_fn)))
for row in f.stdout:
rawRow = row.strip()
row = rawRow.split()
ctgName = row[0]
pos = int(row[1])
if args.bed_fn != None:
if not is_region_in(tree, ctgName, pos):
continue
key = "-".join([ctgName, str(pos)])
if key in d:
continue
if random() < r:
output_fh.stdin.write(rawRow)
output_fh.stdin.write("\n")
o2 += 1
f.stdout.close()
f.wait()
output_fh.stdin.close()
output_fh.wait()
output_fpo.close()
logging.info("%.2f/%.2f Truth Variants/Non-variants outputed" % (o1, o2))
def MergeVcf_illumina(args):
# region vcf merge for illumina, as read realignment will make candidate varaints shift and missing.
bed_fn_prefix = args.bed_fn_prefix
output_fn = args.output_fn
full_alignment_vcf_fn = args.full_alignment_vcf_fn
pileup_vcf_fn = args.pileup_vcf_fn # true vcf var
contig_name = args.ctgName
QUAL = args.qual
bed_fn = None
if not os.path.exists(bed_fn_prefix):
exit(
log_error("[ERROR] Input directory: {} not exists!").format(
bed_fn_prefix))
all_files = os.listdir(bed_fn_prefix)
all_files = [
item for item in all_files if item.startswith(contig_name + '.')
]
if len(all_files) != 0:
bed_fn = os.path.join(bed_fn_prefix,
"full_aln_regions_{}".format(contig_name))
with open(bed_fn, 'w') as output_file:
for file in all_files:
with open(os.path.join(bed_fn_prefix, file)) as f:
output_file.write(f.read())
is_haploid_precise_mode_enabled = args.haploid_precise
is_haploid_sensitive_mode_enabled = args.haploid_sensitive
print_ref = args.print_ref_calls
tree = bed_tree_from(bed_file_path=bed_fn,
padding=param.no_of_positions,
contig_name=contig_name)
unzip_process = subprocess_popen(
shlex.split("gzip -fdc %s" % (pileup_vcf_fn)))
output_dict = {}
header = []
pileup_count = 0
for row in unzip_process.stdout:
if row[0] == '#':
header.append(row)
continue
columns = row.strip().split()
ctg_name = columns[0]
if contig_name != None and ctg_name != contig_name:
continue
pos = int(columns[1])
qual = float(columns[5])
pass_bed = is_region_in(tree, ctg_name, pos)
ref_base, alt_base = columns[3], columns[4]
is_reference = (alt_base == "." or ref_base == alt_base)
if is_haploid_precise_mode_enabled:
row = update_haploid_precise_genotype(columns)
if is_haploid_sensitive_mode_enabled:
row = update_haploid_sensitive_genotype(columns)
if not pass_bed:
if not is_reference:
row = MarkLowQual(row, QUAL, qual)
output_dict[pos] = row
pileup_count += 1
elif print_ref:
output_dict[pos] = row
pileup_count += 1
unzip_process.stdout.close()
unzip_process.wait()
realigned_vcf_unzip_process = subprocess_popen(
shlex.split("gzip -fdc %s" % (full_alignment_vcf_fn)))
realiged_read_num = 0
for row in realigned_vcf_unzip_process.stdout:
if row[0] == '#':
continue
columns = row.strip().split()
ctg_name = columns[0]
if contig_name != None and ctg_name != contig_name:
continue
pos = int(columns[1])
qual = float(columns[5])
ref_base, alt_base = columns[3], columns[4]
is_reference = (alt_base == "." or ref_base == alt_base)
if is_haploid_precise_mode_enabled:
row = update_haploid_precise_genotype(columns)
if is_haploid_sensitive_mode_enabled:
row = update_haploid_sensitive_genotype(columns)
if is_region_in(tree, ctg_name, pos):
if not is_reference:
row = MarkLowQual(row, QUAL, qual)
output_dict[pos] = row
realiged_read_num += 1
elif print_ref:
output_dict[pos] = row
realiged_read_num += 1
logging.info('[INFO] Pileup positions variants proceeded in {}: {}'.format(
contig_name, pileup_count))
logging.info(
'[INFO] Realigned positions variants proceeded in {}: {}'.format(
contig_name, realiged_read_num))
realigned_vcf_unzip_process.stdout.close()
realigned_vcf_unzip_process.wait()
with open(output_fn, 'w') as output_file:
output_list = header + [
output_dict[pos] for pos in sorted(output_dict.keys())
]
output_file.write(''.join(output_list))
def MergeVcf(args):
"""
Merge pileup and full alignment vcf output. We merge the low quality score pileup candidates
recalled by full-alignment model with high quality score pileup output.
"""
output_fn = args.output_fn
full_alignment_vcf_fn = args.full_alignment_vcf_fn
pileup_vcf_fn = args.pileup_vcf_fn # true vcf var
contig_name = args.ctgName
QUAL = args.qual
is_haploid_precise_mode_enabled = args.haploid_precise
is_haploid_sensitive_mode_enabled = args.haploid_sensitive
print_ref = args.print_ref_calls
full_alignment_vcf_unzip_process = subprocess_popen(
shlex.split("gzip -fdc %s" % (full_alignment_vcf_fn)))
full_alignment_output = []
full_alignment_output_set = set()
header = []
for row in full_alignment_vcf_unzip_process.stdout:
if row[0] == '#':
header.append(row)
continue
columns = row.strip().split()
ctg_name = columns[0]
if contig_name != None and ctg_name != contig_name:
continue
pos = int(columns[1])
qual = float(columns[5])
ref_base, alt_base = columns[3], columns[4]
is_reference = (alt_base == "." or ref_base == alt_base)
full_alignment_output_set.add((ctg_name, pos))
if is_haploid_precise_mode_enabled:
row = update_haploid_precise_genotype(columns)
if is_haploid_sensitive_mode_enabled:
row = update_haploid_sensitive_genotype(columns)
if not is_reference:
row = MarkLowQual(row, QUAL, qual)
full_alignment_output.append((pos, row))
elif print_ref:
full_alignment_output.append((pos, row))
full_alignment_vcf_unzip_process.stdout.close()
full_alignment_vcf_unzip_process.wait()
pileup_vcf_unzip_process = subprocess_popen(
shlex.split("gzip -fdc %s" % (pileup_vcf_fn)))
output_file = open(output_fn, 'w')
output_file.write(''.join(header))
def pileup_vcf_generator_from(pileup_vcf_unzip_process):
pileup_row_count = 0
for row in pileup_vcf_unzip_process.stdout:
if row[0] == '#':
continue
columns = row.rstrip().split('\t')
ctg_name = columns[0]
if contig_name and contig_name != ctg_name:
continue
pos = int(columns[1])
qual = float(columns[5])
ref_base, alt_base = columns[3], columns[4]
is_reference = (alt_base == "." or ref_base == alt_base)
if (ctg_name, pos) in full_alignment_output_set:
continue
if is_haploid_precise_mode_enabled:
row = update_haploid_precise_genotype(columns)
if is_haploid_sensitive_mode_enabled:
row = update_haploid_sensitive_genotype(columns)
if not is_reference:
row = MarkLowQual(row, QUAL, qual)
pileup_row_count += 1
yield (pos, row)
elif print_ref:
pileup_row_count += 1
yield (pos, row)
logging.info('[INFO] Pileup variants processed in {}: {}'.format(
contig_name, pileup_row_count))
pileup_vcf_generator = pileup_vcf_generator_from(
pileup_vcf_unzip_process=pileup_vcf_unzip_process)
full_alignment_vcf_generator = iter(full_alignment_output)
for vcf_infos in heapq.merge(full_alignment_vcf_generator,
pileup_vcf_generator):
if len(vcf_infos) != 2:
continue
pos, row = vcf_infos
output_file.write(row)
logging.info('[INFO] Full-alignment variants processed in {}: {}'.format(
contig_name, len(full_alignment_output)))
pileup_vcf_unzip_process.stdout.close()
pileup_vcf_unzip_process.wait()
output_file.close()
def reads_realignment(args):
bed_file_path = args.full_aln_regions
extend_bed = args.extend_bed
fasta_file_path = args.ref_fn
ctg_name = args.ctgName
ctg_start = args.ctgStart
ctg_end = args.ctgEnd
chunk_id = args.chunk_id - 1 if args.chunk_id else None # 1-base to 0-base
chunk_num = args.chunk_num
samtools_execute_command = args.samtools
bam_file_path = args.bam_fn
minMQ = args.minMQ
min_coverage = args.minCoverage
is_bed_file_given = bed_file_path is not None
is_ctg_name_given = ctg_name is not None
read_fn = args.read_fn
global test_pos
test_pos = None
if is_bed_file_given:
candidate_file_path_process = subprocess_popen(
shlex.split("gzip -fdc %s" % (bed_file_path)))
candidate_file_path_output = candidate_file_path_process.stdout
ctg_start, ctg_end = float('inf'), 0
for row in candidate_file_path_output:
row = row.rstrip().split('\t')
if row[0] != ctg_name: continue
position = int(row[1]) + 1
end = int(row[2]) + 1
ctg_start = min(position, ctg_start)
ctg_end = max(end, ctg_end)
candidate_file_path_output.close()
candidate_file_path_process.wait()
if chunk_id is not None:
fai_fn = file_path_from(fasta_file_path,
suffix=".fai",
exit_on_not_found=True,
sep='.')
contig_length = 0
with open(fai_fn, 'r') as fai_fp:
for row in fai_fp:
columns = row.strip().split("\t")
contig_name = columns[0]
if contig_name != ctg_name:
continue
contig_length = int(columns[1])
chunk_size = contig_length // chunk_num + 1 if contig_length % chunk_num else contig_length // chunk_num
ctg_start = chunk_size * chunk_id # 0-base to 1-base
ctg_end = ctg_start + chunk_size
is_ctg_range_given = is_ctg_name_given and ctg_start is not None and ctg_end is not None
# 1-based regions [start, end] (start and end inclusive)
ref_regions = []
reads_regions = []
reference_start, reference_end = None, None
if is_ctg_range_given:
extend_start = ctg_start - max_window_size
extend_end = ctg_end + max_window_size
reads_regions.append(
region_from(ctg_name=ctg_name,
ctg_start=extend_start,
ctg_end=extend_end))
reference_start, reference_end = ctg_start - param.expandReferenceRegion, ctg_end + param.expandReferenceRegion
reference_start = 1 if reference_start < 1 else reference_start
ref_regions.append(
region_from(ctg_name=ctg_name,
ctg_start=reference_start,
ctg_end=reference_end))
elif is_ctg_name_given:
reads_regions.append(region_from(ctg_name=ctg_name))
ref_regions.append(region_from(ctg_name=ctg_name))
reference_start = 1
reference_sequence = reference_sequence_from(
samtools_execute_command=samtools_execute_command,
fasta_file_path=fasta_file_path,
regions=ref_regions)
if reference_sequence is None or len(reference_sequence) == 0:
sys.exit(
"[ERROR] Failed to load reference sequence from file ({}).".format(
fasta_file_path))
tree = bed_tree_from(bed_file_path=bed_file_path)
if is_bed_file_given and ctg_name not in tree:
sys.exit("[ERROR] ctg_name({}) not exists in bed file({}).".format(
ctg_name, bed_file_path))
bed_option = ' -L {}'.format(extend_bed) if extend_bed else ""
bed_option = ' -L {}'.format(
bed_file_path) if is_bed_file_given else bed_option
mq_option = ' -q {}'.format(minMQ) if minMQ > 0 else ""
samtools_view_command = "{} view -h {} {}".format(
samtools_execute_command, bam_file_path,
" ".join(reads_regions)) + mq_option + bed_option
samtools_view_process = subprocess_popen(
shlex.split(samtools_view_command))
if read_fn and read_fn == 'PIPE':
save_file_fp = TensorStdout(sys.stdout)
elif read_fn:
save_file_fp = subprocess_popen(shlex.split(
"{} view -bh - -o {}".format(
samtools_execute_command,
read_fn + ('.{}_{}'.format(ctg_start, ctg_end)
if is_ctg_range_given and not test_pos else ""))),
stdin=PIPE,
stdout=PIPE)
reference_start_0_based = 0 if reference_start is None else (
reference_start - 1)
header = []
add_header = False
aligned_reads = defaultdict()
pileup = defaultdict(lambda: {"X": 0})
samtools_view_generator = samtools_view_generator_from(
samtools_view_process=samtools_view_process,
aligned_reads=aligned_reads,
pileup=pileup,
ctg_name=ctg_name,
reference_sequence=reference_sequence,
reference_start_0_based=reference_start_0_based,
header=header)
pre_aligned_reads = defaultdict()
while True:
chunk_start, chunk_end = next(samtools_view_generator)
if chunk_start is None:
break
if not add_header:
save_file_fp.stdin.write(''.join(header))
add_header = True
variant_allele_list = [[position, pileup[position]["X"]]
for position in list(pileup.keys())]
candidate_position_list = [
(position, support_allele_count)
for position, support_allele_count in variant_allele_list
if support_allele_count >= min_coverage
and position >= chunk_start - region_expansion_in_bp -
1 and position <= chunk_end + region_expansion_in_bp - 1
]
candidate_position_list.sort(key=(lambda x: x[0]))
if not len(aligned_reads) or not len(candidate_position_list):
continue
if len(pre_aligned_reads): # update the read in previous chunk
for read_name, read in pre_aligned_reads.items():
aligned_reads[read_name] = read
region_dict = {}
split_region_size = max_window_size
region_tree = IntervalTree()
for split_idx in range((chunk_end - chunk_start) // split_region_size):
split_start = chunk_start + split_idx * split_region_size - region_expansion_in_bp - 1
split_end = split_start + split_region_size + region_expansion_in_bp * 2 + 1
region_dict[(split_start, split_end)] = []
region_tree.addi(split_start, split_end)
for candidate_position in candidate_position_list:
for region in region_tree.at(candidate_position[0]):
region_dict[(region.begin,
region.end)].append(candidate_position[0])
for key, split_candidate_position_list in region_dict.items():
start_pos, end_pos = None, None
windows = []
read_windows_dict = {}
for pos in split_candidate_position_list:
if start_pos is None:
start_pos = pos
end_pos = pos
elif pos > end_pos + 2 * min_windows_distance:
temp_window = (start_pos - min_windows_distance,
end_pos + min_windows_distance)
windows.append(temp_window)
read_windows_dict[temp_window] = []
start_pos = pos
end_pos = pos
else:
end_pos = pos
if start_pos is not None:
temp_window = (start_pos - min_windows_distance,
end_pos + min_windows_distance)
windows.append(temp_window)
read_windows_dict[temp_window] = []
if not len(windows): continue
windows = sorted(windows, key=lambda x: x[0])
max_window_end = max([item[1] for item in windows])
# #find read windows overlap_pair
for read_name, read in aligned_reads.items():
if read.read_start > max_window_end: continue
argmax_window_idx = find_max_overlap_index(
(read.read_start, read.read_end), windows)
if argmax_window_idx is not None:
read_windows_dict[windows[argmax_window_idx]].append(
read_name)
# realignment
for window in windows:
start_pos, end_pos = window
if end_pos - start_pos > max_window_size: # or (window not in need_align_windows_set):
continue
ref_start = start_pos - reference_start_0_based
ref_end = end_pos - reference_start_0_based
ref = reference_sequence[ref_start:ref_end]
reads = []
low_base_quality_pos_list = []
# pypy binding with ctypes for DBG building
for read_name in read_windows_dict[window]:
read = aligned_reads[read_name]
if (
not read.graph_mq
) or read.read_start > end_pos or read.read_end < start_pos:
continue
reads.append(read.seq)
low_base_quality_pos_list.append(' '.join([
str(bq_idx)
for bq_idx, item in enumerate(read.base_quality)
if int(item) < 15
]))
totoal_read_num = len(reads)
c_ref = byte(ref)
read_list1 = ctypes.c_char_p(byte(','.join(reads)))
low_base_quality_pos_array = ctypes.c_char_p(
byte(','.join(low_base_quality_pos_list)))
dbg.get_consensus.restype = ctypes.POINTER(DBGPointer)
dbg.get_consensus.argtypes = [
ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p,
ctypes.c_int
]
dbg_p = dbg.get_consensus(ctypes.c_char_p(c_ref), read_list1,
low_base_quality_pos_array,
totoal_read_num)
c_consensus, consensus_size = dbg_p.contents.consensus, dbg_p.contents.consensus_size
consensus = [
item.decode() for item in c_consensus[:consensus_size]
]
if len(consensus) == 0 or len(
consensus) == 1 and consensus[0] == ref or len(
read_windows_dict[window]) == 0:
continue
min_read_start = min([
aligned_reads[item].read_start
for item in read_windows_dict[window]
])
max_read_end = max([
aligned_reads[item].read_end
for item in read_windows_dict[window]
])
tmp_ref_start = max(
0,
min(min_read_start, start_pos) - expand_align_ref_region)
tmp_ref_end = max(max_read_end,
end_pos) + expand_align_ref_region
ref_prefix = get_reference_seq(reference_sequence,
tmp_ref_start, start_pos,
reference_start_0_based)
ref_center = get_reference_seq(reference_sequence, start_pos,
end_pos,
reference_start_0_based)
if tmp_ref_end < end_pos:
continue
ref_suffix = get_reference_seq(reference_sequence, end_pos,
tmp_ref_end,
reference_start_0_based)
ref_seq = ref_prefix + ref_center + ref_suffix
# pypy binding with ctypes for realignment
read_name_list = []
totoal_read_num = min(max_region_reads_num,
len(read_windows_dict[window]))
seq_list = (ctypes.c_char_p * totoal_read_num)()
position_list = (ctypes.c_int * totoal_read_num)()
cigars_list = (ctypes.c_char_p * totoal_read_num)()
for read_idx, read_name in enumerate(
read_windows_dict[window]):
read = aligned_reads[read_name]
if read_idx >= totoal_read_num: break
seq_list[read_idx] = byte(read.seq.upper())
position_list[read_idx] = read.read_start
cigars_list[read_idx] = byte(read.cigar)
read_name_list.append(read_name)
haplotypes_list = [
ref_prefix + cons + ref_suffix for cons in consensus
]
haplotypes = ' '.join(haplotypes_list)
realigner.realign_reads.restype = ctypes.POINTER(StructPointer)
realigner.realign_reads.argtypes = [
ctypes.c_char_p * totoal_read_num,
ctypes.c_int * totoal_read_num,
ctypes.c_char_p * totoal_read_num, ctypes.c_char_p,
ctypes.c_char_p, ctypes.c_int, ctypes.c_int, ctypes.c_int,
ctypes.c_int
]
realigner_p = realigner.realign_reads(
seq_list, position_list, cigars_list,
ctypes.c_char_p(byte(ref_seq)),
ctypes.c_char_p(byte(haplotypes)), tmp_ref_start,
len(ref_prefix), len(ref_suffix), totoal_read_num)
realign_positions, realign_cigars = realigner_p.contents.position, realigner_p.contents.cigar_string
read_position_list = realign_positions[:totoal_read_num]
read_cigar_list = [
item.decode() for item in realign_cigars[:totoal_read_num]
]
if len(read_name_list):
for read_id, read_name in enumerate(read_name_list):
if read_cigar_list[read_id] == "" or (
aligned_reads[read_name].cigar
== read_cigar_list[read_id]
and aligned_reads[read_name].read_start
== read_position_list[read_id]):
continue
# update cigar and read start position
aligned_reads[read_name].test_pos = test_pos
realignment_start = read_position_list[read_id]
realignment_cigar = read_cigar_list[read_id].replace(
'X', 'M')
if realignment_cigar == aligned_reads[
read_name].cigar and realignment_start == aligned_reads[
read_name].read_start:
continue
aligned_reads[read_name].set_realignment_info(
split_start, read_cigar_list[read_id],
read_position_list[read_id])
realigner.free_memory.restype = ctypes.POINTER(ctypes.c_void_p)
realigner.free_memory.argtypes = [
ctypes.POINTER(StructPointer), ctypes.c_int
]
realigner.free_memory(realigner_p, totoal_read_num)
# # realignment end
if read_fn:
sorted_key = sorted([(key, item.best_pos)
for key, item in aligned_reads.items()],
key=lambda x: x[1])
for read_name, read_start in sorted_key:
read = aligned_reads[read_name]
if read_start < chunk_start - region_expansion_in_bp - max_window_size: # safe distance for save reads
phasing_info = 'HP:i:{}'.format(
read.phasing) if read.phasing else ""
pass
read_str = '\t'.join([
read_name, read.flag, ctg_name,
str(read_start + 1),
str(read.mapping_quality), read.best_cigar, read.RNEXT,
read.PNEXT, read.TLEN, read.seq, read.raw_base_quality,
phasing_info
])
save_file_fp.stdin.write(read_str + '\n')
del aligned_reads[read_name]
for pile_pos in list(pileup.keys()):
if pile_pos < chunk_start - region_expansion_in_bp - max_window_size:
del pileup[pile_pos]
if read_fn and aligned_reads:
sorted_key = sorted([(key, item.best_pos)
for key, item in aligned_reads.items()],
key=lambda x: x[1])
for read_name, read_start in sorted_key:
read = aligned_reads[read_name]
phasing_info = 'HP:i:{}'.format(
read.phasing) if read.phasing else ""
read_str = '\t'.join([
read_name, read.flag, ctg_name,
str(read_start + 1),
str(read.mapping_quality), read.best_cigar, read.RNEXT,
read.PNEXT, read.TLEN, read.seq, read.raw_base_quality,
phasing_info
])
save_file_fp.stdin.write(read_str + '\n')
del aligned_reads[read_name]
if read_fn != 'PIPE':
save_file_fp.stdin.close()
save_file_fp.wait()
samtools_view_process.stdout.close()
samtools_view_process.wait()
if test_pos:
save_file_fp = subprocess_popen(shlex.split("samtools index {}".format(
read_fn + ('.{}_{}'.format(ctg_start, ctg_end)
if is_ctg_range_given and not test_pos else ""))),
stdin=PIPE,
stdout=PIPE)
save_file_fp.stdin.close()
save_file_fp.wait()
def CreateTensorPileup(args):
"""
Create pileup tensor for pileup model training or calling.
Use slide window to scan the whole candidate regions, keep all candidates over specific minimum allelic frequency
and minimum depth, use samtools mpileup to store pileup info for pileup tensor generation. Only scan candidate
regions once, we could directly get all variant candidates directly.
"""
ctg_start = args.ctgStart
ctg_end = args.ctgEnd
fasta_file_path = args.ref_fn
ctg_name = args.ctgName
samtools_execute_command = args.samtools
bam_file_path = args.bam_fn
chunk_id = args.chunk_id - 1 if args.chunk_id else None # 1-base to 0-base
chunk_num = args.chunk_num
tensor_can_output_path = args.tensor_can_fn
minimum_af_for_candidate = args.min_af
minimum_snp_af_for_candidate = args.snp_min_af
minimum_indel_af_for_candidate = args.indel_min_af
min_coverage = args.minCoverage
platform = args.platform
confident_bed_fn = args.bed_fn
is_confident_bed_file_given = confident_bed_fn is not None
alt_fn = args.indel_fn
extend_bed = args.extend_bed
is_extend_bed_file_given = extend_bed is not None
min_mapping_quality = args.minMQ
min_base_quality = args.minBQ
fast_mode = args.fast_mode
vcf_fn = args.vcf_fn
is_known_vcf_file_provided = vcf_fn is not None
call_snp_only = args.call_snp_only
global test_pos
test_pos = None
# 1-based regions [start, end] (start and end inclusive)
ref_regions = []
reads_regions = []
known_variants_set = set()
tree, bed_start, bed_end = bed_tree_from(bed_file_path=extend_bed,
contig_name=ctg_name,
return_bed_region=True)
fai_fn = file_path_from(fasta_file_path,
suffix=".fai",
exit_on_not_found=True,
sep='.')
if not is_confident_bed_file_given and chunk_id is not None:
contig_length = 0
with open(fai_fn, 'r') as fai_fp:
for row in fai_fp:
columns = row.strip().split("\t")
contig_name = columns[0]
if contig_name != ctg_name:
continue
contig_length = int(columns[1])
chunk_size = contig_length // chunk_num + 1 if contig_length % chunk_num else contig_length // chunk_num
ctg_start = chunk_size * chunk_id # 0-base to 1-base
ctg_end = ctg_start + chunk_size
if is_confident_bed_file_given and chunk_id is not None:
chunk_size = (bed_end - bed_start) // chunk_num + 1 if (
bed_end - bed_start) % chunk_num else (bed_end -
bed_start) // chunk_num
ctg_start = bed_start + 1 + chunk_size * chunk_id # 0-base to 1-base
ctg_end = ctg_start + chunk_size
if is_known_vcf_file_provided and chunk_id is not None:
known_variants_list = vcf_candidates_from(vcf_fn=vcf_fn,
contig_name=ctg_name)
total_variants_size = len(known_variants_list)
chunk_variants_size = total_variants_size // chunk_num if total_variants_size % chunk_num == 0 else total_variants_size // chunk_num + 1
chunk_start_pos = chunk_id * chunk_variants_size
known_variants_set = set(
known_variants_list[chunk_start_pos:chunk_start_pos +
chunk_variants_size])
if len(known_variants_set) == 0:
return
ctg_start, ctg_end = min(known_variants_set), max(known_variants_set)
is_ctg_name_given = ctg_name is not None
is_ctg_range_given = is_ctg_name_given and ctg_start is not None and ctg_end is not None
if is_ctg_range_given:
extend_start = ctg_start - no_of_positions
extend_end = ctg_end + no_of_positions
reads_regions.append(
region_from(ctg_name=ctg_name,
ctg_start=extend_start,
ctg_end=extend_end))
reference_start, reference_end = ctg_start - param.expandReferenceRegion, ctg_end + param.expandReferenceRegion
reference_start = 1 if reference_start < 1 else reference_start
ref_regions.append(
region_from(ctg_name=ctg_name,
ctg_start=reference_start,
ctg_end=reference_end))
elif is_ctg_name_given:
reads_regions.append(region_from(ctg_name=ctg_name))
ref_regions.append(region_from(ctg_name=ctg_name))
reference_start = 1
reference_sequence = reference_sequence_from(
samtools_execute_command=samtools_execute_command,
fasta_file_path=fasta_file_path,
regions=ref_regions)
if reference_sequence is None or len(reference_sequence) == 0:
sys.exit(
log_error(
"[ERROR] Failed to load reference sequence from file ({}).".
format(fasta_file_path)))
if is_confident_bed_file_given and ctg_name not in tree:
sys.exit(
log_error("[ERROR] ctg_name {} not exists in bed file({}).".format(
ctg_name, confident_bed_fn)))
# samtools mpileup options
# reverse-del: deletion in forward/reverse strand were marked as '*'/'#'
min_base_quality = 0 if args.gvcf else min_base_quality
max_depth = param.max_depth_dict[
args.platform] if args.platform else args.max_depth
mq_option = ' --min-MQ {}'.format(min_mapping_quality)
bq_option = ' --min-BQ {}'.format(min_base_quality)
flags_option = ' --excl-flags {}'.format(param.SAMTOOLS_VIEW_FILTER_FLAG)
max_depth_option = ' --max-depth {}'.format(max_depth)
bed_option = ' -l {}'.format(
extend_bed) if is_extend_bed_file_given else ""
gvcf_option = ' -a' if args.gvcf else ""
samtools_mpileup_process = subprocess_popen(
shlex.split("{} mpileup {} -r {} --reverse-del".format(
samtools_execute_command,
bam_file_path,
" ".join(reads_regions),
) + mq_option + bq_option + bed_option + flags_option +
max_depth_option + gvcf_option))
if tensor_can_output_path != "PIPE":
tensor_can_fpo = open(tensor_can_output_path, "wb")
tensor_can_fp = subprocess_popen(shlex.split("{} -c".format(
param.zstd)),
stdin=PIPE,
stdout=tensor_can_fpo)
else:
tensor_can_fp = TensorStdout(sys.stdout)
# whether save all alternative information, only for debug mode
if alt_fn:
alt_fp = open(alt_fn, 'w')
pos_offset = 0
pre_pos = -1
tensor = [[]] * sliding_window_size
candidate_position = []
all_alt_dict = {}
depth_dict = {}
af_dict = {}
# to generate gvcf, it is needed to record whole genome statistical information
if args.gvcf:
nonVariantCaller = variantInfoCalculator(
gvcfWritePath=args.temp_file_dir,
ref_path=args.ref_fn,
bp_resolution=args.bp_resolution,
ctgName=ctg_name,
sample_name='.'.join(
[args.sampleName, ctg_name,
str(ctg_start),
str(ctg_end)]),
p_err=args.base_err,
gq_bin_size=args.gq_bin_size)
confident_bed_tree = bed_tree_from(bed_file_path=confident_bed_fn,
contig_name=ctg_name,
bed_ctg_start=extend_start,
bed_ctg_end=extend_end)
empty_pileup_flag = True
for row in samtools_mpileup_process.stdout:
empty_pileup_flag = False
columns = row.strip().split('\t', maxsplit=5)
pos = int(columns[1])
pileup_bases = columns[4]
reference_base = reference_sequence[pos - reference_start].upper()
valid_reference_flag = True
within_flag = True
if args.gvcf:
if not valid_reference_flag:
nonVariantCaller.make_gvcf_online({}, push_current=True)
if ctg_start != None and ctg_end != None:
within_flag = pos >= ctg_start and pos < ctg_end
elif ctg_start != None and ctg_end == None:
within_flag = pos >= ctg_start
elif ctg_start == None and ctg_end != None:
within_flag = pos <= ctg_end
else:
within_flag = True
if columns[3] == '0' and within_flag and valid_reference_flag:
cur_site_info = {
'chr': columns[0],
'pos': pos,
'ref': reference_base,
'n_total': 0,
'n_ref': 0
}
nonVariantCaller.make_gvcf_online(cur_site_info)
continue
# start with a new region, clear all sliding windows cache, avoid memory occupation
if pre_pos + 1 != pos:
pos_offset = 0
tensor = [[]] * sliding_window_size
candidate_position = []
pre_pos = pos
# a condition to skip some positions creating tensor,but return allele summary
# allele count function
pileup_tensor, alt_dict, af, depth, pass_af, pileup_list, max_del_length = generate_tensor(
pos=pos,
pileup_bases=pileup_bases,
reference_sequence=reference_sequence,
reference_start=reference_start,
reference_base=reference_base,
minimum_af_for_candidate=minimum_af_for_candidate,
minimum_snp_af_for_candidate=minimum_snp_af_for_candidate,
minimum_indel_af_for_candidate=minimum_indel_af_for_candidate,
platform=platform,
fast_mode=fast_mode,
call_snp_only=call_snp_only)
if args.gvcf and within_flag and valid_reference_flag:
cur_n_total = 0
cur_n_ref = 0
for _key, _value in pileup_list:
if (_key == reference_base):
cur_n_ref = _value
cur_n_total += _value
cur_site_info = {
'chr': columns[0],
'pos': pos,
'ref': reference_base,
'n_total': cur_n_total,
'n_ref': cur_n_ref
}
nonVariantCaller.make_gvcf_online(cur_site_info)
pass_confident_bed = not is_confident_bed_file_given or is_region_in(
tree=confident_bed_tree,
contig_name=ctg_name,
region_start=pos - 1,
region_end=pos + max_del_length + 1) # 0-based
if (pass_confident_bed and reference_base in 'ACGT' and
(pass_af and depth >= min_coverage)
and not is_known_vcf_file_provided) or (
is_known_vcf_file_provided and pos in known_variants_set):
candidate_position.append(pos)
all_alt_dict[pos] = alt_dict
depth_dict[pos] = depth
af_dict[pos] = af
tensor[pos_offset] = pileup_tensor
# save pileup tensor for each candidate position with nearby flanking_base_num bp distance
pos_offset = (pos_offset + 1) % sliding_window_size
if len(candidate_position
) and pos - candidate_position[0] == flanking_base_num:
center = candidate_position.pop(0)
has_empty_tensor = sum([True for item in tensor if not len(item)])
if not has_empty_tensor:
depth = depth_dict[center]
ref_seq = reference_sequence[center - (flanking_base_num) -
reference_start:center +
flanking_base_num + 1 -
reference_start]
concat_tensor = tensor[pos_offset:] + tensor[0:pos_offset]
alt_info = str(depth) + '-' + ' '.join([
' '.join([item[0], str(item[1])])
for item in list(all_alt_dict[center].items())
])
l = "%s\t%d\t%s\t%s\t%s" % (
ctg_name, center, ref_seq, " ".join(
" ".join("%d" % x for x in innerlist)
for innerlist in concat_tensor), alt_info)
tensor_can_fp.stdin.write(l)
tensor_can_fp.stdin.write("\n")
if alt_fn:
alt_info = ' '.join([
' '.join([item[0], str(item[1])])
for item in list(all_alt_dict[center].items())
])
alt_fp.write('\t'.join([
ctg_name + ' ' + str(center),
str(depth), alt_info,
str(af_dict[center])
]) + '\n')
del all_alt_dict[center], depth_dict[center], af_dict[center]
if args.gvcf and len(nonVariantCaller.current_block) != 0:
nonVariantCaller.write_to_gvcf_batch(nonVariantCaller.current_block,
nonVariantCaller.cur_min_DP,
nonVariantCaller.cur_raw_gq)
if args.gvcf and empty_pileup_flag:
nonVariantCaller.write_empty_pileup(ctg_name, ctg_start, ctg_end)
if args.gvcf:
nonVariantCaller.close_vcf_writer()
samtools_mpileup_process.stdout.close()
samtools_mpileup_process.wait()
if tensor_can_output_path != "PIPE":
tensor_can_fp.stdin.close()
tensor_can_fp.wait()
tensor_can_fpo.close()
if alt_fn:
alt_fp.close()
def Run(args):
basedir = dirname(__file__)
EVCBin = basedir + "/../clair.py ExtractVariantCandidates"
GTBin = basedir + "/../clair.py GetTruth"
CTBin = basedir + "/../clair.py CreateTensor"
CVBin = basedir + "/../clair.py call_var"
pypyBin = executable_command_string_from(args.pypy, exit_on_not_found=True)
samtoolsBin = executable_command_string_from(args.samtools,
exit_on_not_found=True)
chkpnt_fn = file_path_from(args.chkpnt_fn,
suffix=".meta",
exit_on_not_found=True)
bam_fn = file_path_from(args.bam_fn, exit_on_not_found=True)
ref_fn = file_path_from(args.ref_fn, exit_on_not_found=True)
vcf_fn = file_path_from(args.vcf_fn)
bed_fn = file_path_from(args.bed_fn)
dcov = args.dcov
call_fn = args.call_fn
af_threshold = args.threshold
minCoverage = int(args.minCoverage)
sampleName = args.sampleName
ctgName = args.ctgName
if ctgName is None:
sys.exit(
"--ctgName must be specified. You can call variants on multiple chromosomes simultaneously."
)
stop_consider_left_edge = command_option_from(args.stop_consider_left_edge,
'stop_consider_left_edge')
log_path = command_option_from(args.log_path,
'log_path',
option_value=args.log_path)
pysam_for_all_indel_bases = command_option_from(
args.pysam_for_all_indel_bases, 'pysam_for_all_indel_bases')
haploid_precision_mode = command_option_from(args.haploid_precision,
'haploid_precision')
haploid_sensitive_mode = command_option_from(args.haploid_sensitive,
'haploid_sensitive')
output_for_ensemble = command_option_from(args.output_for_ensemble,
'output_for_ensemble')
pipe_line = command_option_from(args.pipe_line, 'pipe_line')
store_loaded_mini_match = command_option_from(args.store_loaded_mini_match,
'store_loaded_mini_match')
only_prediction = command_option_from(args.only_prediction,
'only_prediction')
debug = command_option_from(args.debug, 'debug')
qual = command_option_from(args.qual, 'qual', option_value=args.qual)
fast_plotting = command_option_from(args.fast_plotting, 'fast_plotting')
ctgStart = None
ctgEnd = None
if args.ctgStart is not None and args.ctgEnd is not None and int(
args.ctgStart) <= int(args.ctgEnd):
ctgStart = CommandOption('ctgStart', args.ctgStart)
ctgEnd = CommandOption('ctgEnd', args.ctgEnd)
if args.threads is None:
numCpus = multiprocessing.cpu_count()
else:
numCpus = args.threads if args.threads < multiprocessing.cpu_count(
) else multiprocessing.cpu_count()
maxCpus = multiprocessing.cpu_count()
_cpuSet = ",".join(
str(x) for x in random.sample(range(0, maxCpus), numCpus))
taskSet = "taskset -c %s" % (_cpuSet)
try:
subprocess.check_output("which %s" % ("taskset"), shell=True)
except:
taskSet = ""
if args.delay > 0:
delay = random.randrange(0, args.delay)
print("Delay %d seconds before starting variant calling ..." % (delay),
file=sys.stderr)
sleep(delay)
extract_variant_candidate_command_options = [
pypyBin, EVCBin,
CommandOption('bam_fn', bam_fn),
CommandOption('ref_fn', ref_fn),
CommandOption('bed_fn', bed_fn),
CommandOption('ctgName', ctgName), ctgStart, ctgEnd,
CommandOption('threshold', af_threshold),
CommandOption('minCoverage', minCoverage),
CommandOption('samtools', samtoolsBin)
]
get_truth_command_options = [
pypyBin, GTBin,
CommandOption('vcf_fn', vcf_fn),
CommandOption('ref_fn', ref_fn),
CommandOption('ctgName', ctgName), ctgStart, ctgEnd
]
create_tensor_command_options = [
pypyBin, CTBin,
CommandOption('bam_fn', bam_fn),
CommandOption('ref_fn', ref_fn),
CommandOption('ctgName', ctgName), ctgStart, ctgEnd,
stop_consider_left_edge,
CommandOption('samtools', samtoolsBin),
CommandOption('dcov', dcov)
]
call_variant_command_options = [
taskSet,
ExecuteCommand('python', CVBin),
CommandOption('chkpnt_fn', chkpnt_fn),
CommandOption('call_fn', call_fn),
CommandOption('bam_fn', bam_fn),
CommandOption('sampleName', sampleName),
CommandOption('time_counter_file_name', args.time_counter_file_name),
CommandOption('threads', numCpus),
CommandOption('ref_fn', ref_fn), pysam_for_all_indel_bases,
haploid_precision_mode, haploid_sensitive_mode, output_for_ensemble,
pipe_line, store_loaded_mini_match, only_prediction, qual, debug
]
call_variant_with_activation_command_options = [
CommandOptionWithNoValue('activation_only'),
log_path,
CommandOption('max_plot', args.max_plot),
CommandOption('parallel_level', args.parallel_level),
CommandOption('workers', args.workers),
fast_plotting,
] if args.activation_only else []
is_true_variant_call = vcf_fn is not None
try:
c.extract_variant_candidate = subprocess_popen(
shlex.split(
command_string_from(
get_truth_command_options if is_true_variant_call else
extract_variant_candidate_command_options)))
c.create_tensor = subprocess_popen(
shlex.split(command_string_from(create_tensor_command_options)),
stdin=c.extract_variant_candidate.stdout)
c.call_variant = subprocess_popen(shlex.split(
command_string_from(call_variant_command_options +
call_variant_with_activation_command_options)),
stdin=c.create_tensor.stdout,
stdout=sys.stderr)
except Exception as e:
print(e, file=sys.stderr)
sys.exit("Failed to start required processes. Exiting...")
signal.signal(signal.SIGALRM, check_return_code)
signal.alarm(2)
try:
c.call_variant.wait()
c.create_tensor.stdout.close()
c.create_tensor.wait()
c.extract_variant_candidate.stdout.close()
c.extract_variant_candidate.wait()
except KeyboardInterrupt as e:
print(
"KeyboardInterrupt received when waiting at CallVarBam, terminating all scripts."
)
try:
c.call_variant.terminate()
c.create_tensor.terminate()
c.extract_variant_candidate.terminate()
except Exception as e:
print(e)
raise KeyboardInterrupt
except Exception as e:
print(
"Exception received when waiting at CallVarBam, terminating all scripts."
)
print(e)
try:
c.call_variant.terminate()
c.create_tensor.terminate()
c.extract_variant_candidate.terminate()
except Exception as e:
print(e)
raise e
def SelectCandidates(args):
"""
Select low quality and low sequence entropy candidate variants for full aligement. False positive pileup variants
and true variants missed by pileup calling would mostly have low quality score (reference quality score for missing
variants), so only use a proportion of low quality variants for full alignment while maintain high quality pileup
output, as full alignment calling is substantially slower than pileup calling.
"""
phased_vcf_fn = args.phased_vcf_fn
pileup_vcf_fn = args.pileup_vcf_fn
var_pct_full = args.var_pct_full
ref_pct_full = args.ref_pct_full
seq_entropy_pro = args.seq_entropy_pro
contig_name = args.ctgName
phasing_window_size = param.phasing_window_size
platform = args.platform
split_bed_size = args.split_bed_size
split_folder = args.split_folder
extend_bp = param.extend_bp
call_low_seq_entropy = args.call_low_seq_entropy
phasing_info_in_bam = args.phasing_info_in_bam
need_phasing_list = []
need_phasing_set = set()
ref_call_pos_list = []
variant_dict = defaultdict(str)
flankingBaseNum = param.flankingBaseNum
qual_fn = args.qual_fn if args.qual_fn is not None else 'qual'
fasta_file_path = args.ref_fn
samtools_execute_command = args.samtools
found_qual_cut_off = False
low_sequence_entropy_list = []
# try to find the global quality cut off:
f_qual = os.path.join(split_folder, qual_fn)
if os.path.exists(f_qual):
with open(f_qual, 'r') as f:
line = f.read().rstrip().split(' ')
var_qual, ref_qual = float(line[0]), float(line[1])
found_qual_cut_off = True
all_full_aln_regions = []
if phased_vcf_fn and os.path.exists(phased_vcf_fn):
unzip_process = subprocess_popen(
shlex.split("gzip -fdc %s" % (phased_vcf_fn)))
for row in unzip_process.stdout:
row = row.rstrip()
if row[0] == '#':
continue
columns = row.strip().split('\t')
ctg_name = columns[0]
if contig_name and contig_name != ctg_name:
continue
pos = int(columns[1])
ref_base = columns[3]
alt_base = columns[4]
genotype_info = columns[9].split(':')
genotype, phase_set = genotype_info[0], genotype_info[-1]
if '|' not in genotype: # unphasable
continue
variant_dict[pos] = '-'.join([
ref_base, alt_base, ('1' if genotype == '0|1' else '2'),
phase_set
])
if pileup_vcf_fn and os.path.exists(pileup_vcf_fn):
# vcf format
unzip_process = subprocess_popen(
shlex.split("gzip -fdc %s" % (pileup_vcf_fn)))
for row in unzip_process.stdout:
if row[0] == '#':
continue
columns = row.rstrip().split('\t')
ctg_name = columns[0]
if contig_name and contig_name != ctg_name:
continue
pos = int(columns[1])
ref_base = columns[3]
alt_base = columns[4]
qual = float(columns[5])
# reference calling
if alt_base == "." or ref_base == alt_base:
ref_call_pos_list.append((pos, qual))
else:
need_phasing_list.append((pos, qual))
need_phasing_set.add(pos)
if found_qual_cut_off:
low_qual_ref_list = [[k, v] for k, v in ref_call_pos_list
if v < ref_qual]
low_qual_variant_list = [[k, v] for k, v in need_phasing_list
if v < var_qual]
else:
low_qual_ref_list = sorted(
ref_call_pos_list,
key=lambda x: x[1])[:int(ref_pct_full *
len(ref_call_pos_list))]
low_qual_variant_list = sorted(
need_phasing_list,
key=lambda x: x[1])[:int(var_pct_full *
len(need_phasing_list))]
if call_low_seq_entropy:
candidate_positions = sorted(
ref_call_pos_list, key=lambda x: x[1])[:int(
(var_pct_full + seq_entropy_pro) * len(ref_call_pos_list)
)] + sorted(need_phasing_list, key=lambda x: x[1])[:int(
(var_pct_full + seq_entropy_pro) * len(need_phasing_list))]
candidate_positions = set(
[item[0] for item in candidate_positions])
candidate_positions_entropy_list = sqeuence_entropy_from(
samtools_execute_command=samtools_execute_command,
fasta_file_path=fasta_file_path,
contig_name=contig_name,
candidate_positions=candidate_positions)
low_sequence_entropy_list = sorted(
candidate_positions_entropy_list, key=lambda x: x[1]
)[:int(seq_entropy_pro * len(candidate_positions_entropy_list))]
# calling with phasing_info_in_bam: select low qual ref and low qual vairant for phasing calling
if phasing_info_in_bam:
logging.info(
'[INFO] Low quality reference calls to be processed in {}: {}'.
format(contig_name, len(low_qual_ref_list)))
logging.info(
'[INFO] Low quality variants to be processed in {}: {}'.format(
contig_name, len(low_qual_variant_list)))
if call_low_seq_entropy:
logging.info(
'[INFO] Total low sequence entropy variants to be processed in {}: {}'
.format(contig_name, len(low_sequence_entropy_list)))
need_phasing_row_list = set(
[item[0] for item in low_qual_ref_list] +
[item[0] for item in low_qual_variant_list] +
[item[0] for item in low_sequence_entropy_list])
need_phasing_row_list = sorted(list(need_phasing_row_list))
if len(need_phasing_row_list) == 0:
print(
log_warning(
"[WARNING] Cannot find any low-quality 0/0, 0/1 or 1/1 variant in pileup output in contig {}"
.format(contig_name)))
region_num = len(
need_phasing_row_list) // split_bed_size + 1 if len(
need_phasing_row_list) % split_bed_size else len(
need_phasing_row_list) // split_bed_size
for idx in range(region_num):
# a windows region for create tensor # samtools mpileup not include last position
split_output = need_phasing_row_list[idx *
split_bed_size:(idx + 1) *
split_bed_size]
if platform == 'ilmn':
region_size = param.split_region_size
split_output = [(item // region_size * region_size -
param.no_of_positions,
item // region_size * region_size +
region_size + param.no_of_positions)
for item in split_output]
else:
split_output = [(item - flankingBaseNum,
item + flankingBaseNum + 2)
for item in split_output]
split_output = sorted(split_output, key=lambda x: x[0])
# currently deprecate using ctgName.start_end as file name, which will run similar regions for several times when start and end has slight difference
# output_path = os.path.join(split_folder, '{}.{}_{}'.format(contig_name, split_output[0][0], split_output[-1][1]))
output_path = os.path.join(
split_folder, '{}.{}_{}'.format(contig_name, idx,
region_num))
all_full_aln_regions.append(output_path)
with open(output_path, 'w') as output_file:
output_file.write('\n'.join([
'\t'.join([
contig_name,
str(x[0] - 1),
str(x[1] - 1),
]) for x in split_output
]) + '\n') # bed format
if len(all_full_aln_regions) > 0:
all_full_aln_regions_path = os.path.join(
split_folder, 'FULL_ALN_FILE_{}'.format(contig_name))
with open(all_full_aln_regions_path, 'w') as output_file:
output_file.write('\n'.join(all_full_aln_regions) + '\n')
return
for pos, qual in low_qual_ref_list:
need_phasing_set.add(pos)
# Call variant in all candidate position
elif args.all_alt_fn is not None:
unzip_process = subprocess_popen(
shlex.split("gzip -fdc %s" % (args.all_alt_fn)))
for row in unzip_process.stdout:
if row[0] == '#':
continue
columns = row.rstrip().split('\t')
ctg_name, pos = columns[0].split()
pos = int(pos)
if contig_name and contig_name != ctg_name:
continue
need_phasing_set.add(pos)
need_phasing_row_list = sorted(list(set(need_phasing_set)))
snp_tree = IntervalTree()
hete_snp_row_list = sorted(
list(
set(variant_dict.keys()).intersection(set(need_phasing_row_list))))
print(
'[INFO] Total hete snp with reads support in {}: '.format(contig_name),
len(hete_snp_row_list))
print(
'[INFO] Total candidates need to be processed in {}: '.format(
contig_name), len(need_phasing_row_list))
for item in hete_snp_row_list:
snp_tree.addi(item, item + 1)
region_num = len(need_phasing_row_list) // split_bed_size + 1 if len(
need_phasing_row_list) % split_bed_size else len(
need_phasing_row_list) // split_bed_size
for idx in range(region_num):
split_output = need_phasing_row_list[idx * split_bed_size:(idx + 1) *
split_bed_size]
start = split_output[0]
end = split_output[-1]
extend_start, extend_end = start - phasing_window_size, end + phasing_window_size
overlaps = snp_tree.overlap(extend_start, extend_end)
snp_split_out = []
for overlap in overlaps:
snp_split_out.append((contig_name, overlap[0] - extend_bp - 1 - 1,
overlap[0] + 1 + extend_bp - 1,
variant_dict[overlap[0]])) # bed format
split_output = [(contig_name, item - flankingBaseNum - 1,
item + flankingBaseNum + 1 - 1)
for item in split_output
] # a windows region for create tensor # bed format
split_output += snp_split_out
split_output = sorted(split_output, key=lambda x: x[1])
with open(
os.path.join(split_folder,
'{}.{}_{}'.format(contig_name, start, end)),
'w') as output_file:
output_file.write(
'\n'.join(['\t'.join(map(str, x))
for x in split_output]) + '\n') # bed format
def OutputVariant(args):
var_fn = args.var_fn
vcf_fn = args.vcf_fn
truth_vcf_fn = args.truth_vcf_fn
ctg_name = args.ctgName
ctg_start = args.ctgStart
ctg_end = args.ctgEnd
truth_vcf_set = set()
variant_set = set()
if args.truth_vcf_fn is not None:
truth_vcf_set = set(
vcf_candidates_from(vcf_fn=truth_vcf_fn, contig_name=ctg_name))
if args.var_fn != "PIPE":
var_fpo = open(var_fn, "wb")
var_fp = subprocess_popen(shlex.split("gzip -c"),
stdin=PIPE,
stdout=var_fpo)
else:
var_fp = TruthStdout(sys.stdout)
is_ctg_region_provided = ctg_start is not None and ctg_end is not None
vcf_fp = subprocess_popen(shlex.split("gzip -fdc %s" % (vcf_fn)))
for row in vcf_fp.stdout:
columns = row.strip().split()
if columns[0][0] == "#":
continue
# position in vcf is 1-based
chromosome, position = columns[0], columns[1]
if chromosome != ctg_name:
continue
if is_ctg_region_provided and not (ctg_start <= int(position) <=
ctg_end):
continue
reference, alternate, last_column = columns[3], columns[4], columns[-1]
# normal GetTruth
genotype = last_column.split(":")[0].replace("/", "|").replace(
".", "0").split("|")
genotype_1, genotype_2 = genotype
# 1000 Genome GetTruth (format problem) (no genotype is given)
if int(genotype_1) > int(genotype_2):
genotype_1, genotype_2 = genotype_2, genotype_1
#remove * to guarentee vcf match
if '*' in alternate:
alternate = alternate.split(',')
if int(genotype_1) + int(genotype_2) != 3 or len(alternate) != 2:
print('error with variant represatation')
continue
alternate = ''.join(
[alt_base for alt_base in alternate if alt_base != '*'])
# * always have a genotype 1/2
genotype_1, genotype_2 = '0', '1'
variant_set.add(int(position))
var_fp.stdin.write(" ".join((chromosome, position, reference,
alternate, genotype_1, genotype_2)))
var_fp.stdin.write("\n")
for position in truth_vcf_set:
if position not in variant_set:
# miss variant set used in Tensor2Bin
var_fp.stdin.write(" ".join(
(chromosome, str(position), "None", "None", "-1", "-1")))
var_fp.stdin.write("\n")
vcf_fp.stdout.close()
vcf_fp.wait()
if args.var_fn != "PIPE":
var_fp.stdin.close()
var_fp.wait()
var_fpo.close()
def make_candidates(args):
gen4Training = args.gen4Training
variant_file_path = args.var_fn
bed_file_path = args.bed_fn
fasta_file_path = args.ref_fn
ctg_name = args.ctgName
ctg_start = args.ctgStart
ctg_end = args.ctgEnd
output_probability = args.outputProb
samtools_execute_command = args.samtools
minimum_depth_for_candidate = args.minCoverage
minimum_af_for_candidate = args.threshold
minimum_mapping_quality = args.minMQ
bam_file_path = args.bam_fn
candidate_output_path = args.can_fn
is_using_stdout_for_output_candidate = candidate_output_path == "PIPE"
is_building_training_dataset = gen4Training == True
is_variant_file_given = variant_file_path is not None
is_bed_file_given = bed_file_path is not None
is_ctg_name_given = ctg_name is not None
is_ctg_range_given = is_ctg_name_given and ctg_start is not None and ctg_end is not None
if is_building_training_dataset:
# minimum_depth_for_candidate = 0
minimum_af_for_candidate = 0
# preparation for candidates near variants
need_consider_candidates_near_variant = is_building_training_dataset and is_variant_file_given
variants_map = variants_map_from(
variant_file_path) if need_consider_candidates_near_variant else {}
non_variants_map = non_variants_map_near_variants_from(variants_map)
no_of_candidates_near_variant = 0
no_of_candidates_outside_variant = 0
# update output probabilities for candidates near variants
# original: (7000000.0 * 2.0 / 3000000000)
ratio_of_candidates_near_variant_to_candidates_outside_variant = 1.0
output_probability_near_variant = (
3500000.0 *
ratio_of_candidates_near_variant_to_candidates_outside_variant *
RATIO_OF_NON_VARIANT_TO_VARIANT / 14000000)
output_probability_outside_variant = 3500000.0 * RATIO_OF_NON_VARIANT_TO_VARIANT / (
3000000000 - 14000000)
if not isfile("{}.fai".format(fasta_file_path)):
print("Fasta index {}.fai doesn't exist.".format(fasta_file_path),
file=sys.stderr)
sys.exit(1)
# 1-based regions [start, end] (start and end inclusive)
regions = []
reference_start, reference_end = None, None
if is_ctg_range_given:
reference_start, reference_end = ctg_start - param.expandReferenceRegion, ctg_end + param.expandReferenceRegion
reference_start = 1 if reference_start < 1 else reference_start
regions.append(
region_from(ctg_name=ctg_name,
ctg_start=reference_start,
ctg_end=reference_end))
elif is_ctg_name_given:
regions.append(region_from(ctg_name=ctg_name))
reference_sequence = reference_sequence_from(
samtools_execute_command=samtools_execute_command,
fasta_file_path=fasta_file_path,
regions=regions)
if reference_sequence is None or len(reference_sequence) == 0:
print(
"[ERROR] Failed to load reference seqeunce from file ({}).".format(
fasta_file_path),
file=sys.stderr)
sys.exit(1)
tree = bed_tree_from(bed_file_path=bed_file_path)
if is_bed_file_given and ctg_name not in tree:
print("[ERROR] ctg_name({}) not exists in bed file({}).".format(
ctg_name, bed_file_path),
file=sys.stderr)
sys.exit(1)
samtools_view_process = subprocess_popen(
shlex.split("{} view -F {} {} {}".format(
samtools_execute_command, param.SAMTOOLS_VIEW_FILTER_FLAG,
bam_file_path, " ".join(regions))))
if is_using_stdout_for_output_candidate:
can_fp = CandidateStdout(sys.stdout)
else:
can_fpo = open(candidate_output_path, "wb")
can_fp = subprocess_popen(shlex.split("gzip -c"),
stdin=PIPE,
stdout=can_fpo)
pileup = defaultdict(lambda: {
"A": 0,
"C": 0,
"G": 0,
"T": 0,
"I": 0,
"D": 0,
"N": 0
})
POS = 0
number_of_reads_processed = 0
while True:
row = samtools_view_process.stdout.readline()
is_finish_reading_output = row == '' and samtools_view_process.poll(
) is not None
if row:
columns = row.strip().split()
if columns[0][0] == "@":
continue
RNAME = columns[2]
if RNAME != ctg_name:
continue
POS = int(
columns[3]
) - 1 # switch from 1-base to 0-base to match sequence index
MAPQ = int(columns[4])
CIGAR = columns[5]
SEQ = columns[9].upper(
) # uppercase for SEQ (regexp is \*|[A-Za-z=.]+)
reference_position = POS
query_position = 0
if MAPQ < minimum_mapping_quality:
continue
if CIGAR == "*" or is_too_many_soft_clipped_bases_for_a_read_from(
CIGAR):
continue
number_of_reads_processed += 1
advance = 0
for c in str(CIGAR):
if c.isdigit():
advance = advance * 10 + int(c)
continue
if c == "S":
query_position += advance
elif c == "M" or c == "=" or c == "X":
for _ in range(advance):
base = evc_base_from(SEQ[query_position])
pileup[reference_position][base] += 1
# those CIGAR operations consumes query and reference
reference_position += 1
query_position += 1
elif c == "I":
pileup[reference_position - 1]["I"] += 1
# insertion consumes query
query_position += advance
elif c == "D":
pileup[reference_position - 1]["D"] += 1
# deletion consumes reference
reference_position += advance
# reset advance
advance = 0
positions = [x for x in pileup.keys() if x < POS
] if not is_finish_reading_output else list(pileup.keys())
positions.sort()
for zero_based_position in positions:
base_count = depth = reference_base = temp_key = None
# ctg and bed checking (region [ctg_start, ctg_end] is 1-based, inclusive start and end positions)
pass_ctg = not is_ctg_range_given or ctg_start <= zero_based_position + 1 <= ctg_end
pass_bed = not is_bed_file_given or is_region_in(
tree, ctg_name, zero_based_position)
if not pass_bed or not pass_ctg:
continue
# output probability checking
pass_output_probability = True
if is_building_training_dataset and is_variant_file_given:
temp_key = ctg_name + ":" + str(zero_based_position + 1)
pass_output_probability = (temp_key not in variants_map and (
(temp_key in non_variants_map and
random.uniform(0, 1) <= output_probability_near_variant)
or (temp_key not in non_variants_map and random.uniform(
0, 1) <= output_probability_outside_variant)))
elif is_building_training_dataset:
pass_output_probability = random.uniform(
0, 1) <= output_probability
if not pass_output_probability:
continue
# for depth checking and af checking
try:
reference_base = evc_base_from(reference_sequence[
zero_based_position -
(0 if reference_start is None else (reference_start - 1))])
position_dict = pileup[zero_based_position]
except:
continue
# depth checking
base_count = list(position_dict.items())
depth = sum(
x[1]
for x in base_count) - position_dict["I"] - position_dict["D"]
if depth < minimum_depth_for_candidate:
continue
# af checking
denominator = depth if depth > 0 else 1
base_count.sort(
key=lambda x: -x[1]) # sort base_count descendingly
pass_af = (base_count[0][0] != reference_base
or (float(base_count[1][1]) / denominator) >=
minimum_af_for_candidate)
if not pass_af:
continue
# output 1-based candidate
if temp_key is not None and temp_key in non_variants_map:
no_of_candidates_near_variant += 1
elif temp_key is not None and temp_key not in non_variants_map:
no_of_candidates_outside_variant += 1
output = [ctg_name, zero_based_position + 1, reference_base, depth]
output.extend(["%s %d" % x for x in base_count])
output = " ".join([str(x) for x in output]) + "\n"
can_fp.stdin.write(output)
for zero_based_position in positions:
del pileup[zero_based_position]
if is_finish_reading_output:
break
if need_consider_candidates_near_variant:
print("# of candidates near variant: ", no_of_candidates_near_variant)
print("# of candidates outside variant: ",
no_of_candidates_outside_variant)
samtools_view_process.stdout.close()
samtools_view_process.wait()
if not is_using_stdout_for_output_candidate:
can_fp.stdin.close()
can_fp.wait()
can_fpo.close()
if number_of_reads_processed == 0:
print(
"No read has been process, either the genome region you specified has no read cover, or please check the correctness of your BAM input (%s)."
% (bam_file_path),
file=sys.stderr)
sys.exit(0)
def OutputAlnTensor(args):
available_slots = 5000000
samtools = args.samtools
tensor_file_path = args.tensor_fn
bam_file_path = args.bam_fn
reference_file_path = args.ref_fn
candidate_file_path = args.can_fn
dcov = args.dcov
is_consider_left_edge = not args.stop_consider_left_edge
min_coverage = args.minCoverage
minimum_mapping_quality = args.minMQ
ctg_name = args.ctgName
ctg_start = args.ctgStart
ctg_end = args.ctgEnd
reference_result = reference_result_from(
ctg_name=ctg_name,
ctg_start=ctg_start,
ctg_end=ctg_end,
samtools=samtools,
reference_file_path=reference_file_path,
expand_reference_region=param.expandReferenceRegion,
)
reference_sequence = reference_result.sequence if reference_result is not None else ""
is_faidx_process_have_error = reference_result is None or reference_result.is_faidx_process_have_error
have_reference_sequence = reference_result is not None and len(reference_sequence) > 0
if reference_result is None or is_faidx_process_have_error or not have_reference_sequence:
print("Failed to load reference seqeunce. Please check if the provided reference fasta %s and the ctgName %s are correct." % (
reference_file_path,
ctg_name
), file=sys.stderr)
sys.exit(1)
reference_start = reference_result.start
reference_start_0_based = 0 if reference_start is None else (reference_start - 1)
begin_to_end = {}
candidate_position = 0
candidate_position_generator = candidate_position_generator_from(
candidate_file_path=candidate_file_path,
ctg_start=ctg_start,
ctg_end=ctg_end,
is_consider_left_edge=is_consider_left_edge,
flanking_base_num=param.flankingBaseNum,
begin_to_end=begin_to_end
)
samtools_view_process = samtools_view_process_from(
ctg_name=ctg_name,
ctg_start=ctg_start,
ctg_end=ctg_end,
samtools=samtools,
bam_file_path=bam_file_path
)
center_to_alignment = {}
if tensor_file_path != "PIPE":
tensor_fpo = open(tensor_file_path, "wb")
tensor_fp = subprocess_popen(shlex.split("gzip -c"), stdin=PIPE, stdout=tensor_fpo)
else:
tensor_fp = TensorStdout(sys.stdout)
previous_position = 0
depthCap = 0
for l in samtools_view_process.stdout:
l = l.split()
if l[0][0] == "@":
continue
FLAG = int(l[1])
POS = int(l[3]) - 1 # switch from 1-base to 0-base to match sequence index
MQ = int(l[4])
CIGAR = l[5]
SEQ = l[9].upper() # uppercase for SEQ (regexp is \*|[A-Za-z=.]+)
reference_position = POS
query_position = 0
STRAND = (16 == (FLAG & 16))
if MQ < minimum_mapping_quality:
continue
end_to_center = {}
active_set = set()
while candidate_position != -1 and candidate_position < (POS + len(SEQ) + 100000):
candidate_position = next(candidate_position_generator)
if previous_position != POS:
previous_position = POS
depthCap = 0
else:
depthCap += 1
if depthCap >= dcov:
#print >> sys.stderr, "Bypassing POS %d at depth %d\n" % (POS, depthCap)
continue
advance = 0
for c in str(CIGAR):
if available_slots <= 0:
break
if c.isdigit():
advance = advance * 10 + int(c)
continue
# soft clip
if c == "S":
query_position += advance
# match / mismatch
if c == "M" or c == "=" or c == "X":
for _ in range(advance):
if reference_position in begin_to_end:
for rEnd, rCenter in begin_to_end[reference_position]:
if rCenter in active_set:
continue
end_to_center[rEnd] = rCenter
active_set.add(rCenter)
center_to_alignment.setdefault(rCenter, [])
center_to_alignment[rCenter].append([])
for center in list(active_set):
if available_slots <= 0:
break
available_slots -= 1
center_to_alignment[center][-1].append((
reference_position,
0,
reference_sequence[reference_position - reference_start_0_based],
SEQ[query_position],
STRAND
))
if reference_position in end_to_center:
center = end_to_center[reference_position]
active_set.remove(center)
reference_position += 1
query_position += 1
# insertion
if c == "I":
for queryAdv in range(advance):
for center in list(active_set):
if available_slots <= 0:
break
available_slots -= 1
center_to_alignment[center][-1].append((
reference_position,
queryAdv,
"-",
SEQ[query_position],
STRAND
))
query_position += 1
# deletion
if c == "D":
for _ in range(advance):
for center in list(active_set):
if available_slots <= 0:
break
available_slots -= 1
center_to_alignment[center][-1].append((
reference_position,
0,
reference_sequence[reference_position - reference_start_0_based],
"-",
STRAND
))
if reference_position in begin_to_end:
for rEnd, rCenter in begin_to_end[reference_position]:
if rCenter in active_set:
continue
end_to_center[rEnd] = rCenter
active_set.add(rCenter)
center_to_alignment.setdefault(rCenter, [])
center_to_alignment[rCenter].append([])
if reference_position in end_to_center:
center = end_to_center[reference_position]
active_set.remove(center)
reference_position += 1
# reset advance
advance = 0
if depthCap == 0:
for center in list(center_to_alignment.keys()):
if center + (param.flankingBaseNum + 1) >= POS:
continue
l = generate_tensor(
ctg_name, center_to_alignment[center], center, reference_sequence, reference_start_0_based, min_coverage
)
if l != None:
tensor_fp.stdin.write(l)
tensor_fp.stdin.write("\n")
available_slots += sum(len(i) for i in center_to_alignment[center])
#print >> sys.stderr, "POS %d: remaining slots %d" % (center, available_slots)
del center_to_alignment[center]
for center in center_to_alignment.keys():
l = generate_tensor(
ctg_name, center_to_alignment[center], center, reference_sequence, reference_start_0_based, min_coverage
)
if l != None:
tensor_fp.stdin.write(l)
tensor_fp.stdin.write("\n")
samtools_view_process.stdout.close()
samtools_view_process.wait()
if tensor_file_path != "PIPE":
tensor_fp.stdin.close()
tensor_fp.wait()
tensor_fpo.close()
def sort_vcf_from(args):
"""
Sort vcf file from providing vcf filename prefix.
"""
output_fn = args.output_fn
input_dir = args.input_dir
vcf_fn_prefix = args.vcf_fn_prefix
vcf_fn_suffix = args.vcf_fn_suffix
sample_name = args.sampleName
ref_fn = args.ref_fn
contigs_fn = args.contigs_fn
if not os.path.exists(input_dir):
exit(
log_error("[ERROR] Input directory: {} not exists!").format(
input_dir))
all_files = os.listdir(input_dir)
if vcf_fn_prefix is not None:
all_files = [
item for item in all_files if item.startswith(vcf_fn_prefix)
]
if len(all_files) == 0:
output_header(output_fn=output_fn,
reference_file_path=ref_fn,
sample_name=sample_name)
print(
log_warning(
"[WARNING] No vcf file found with prefix:{}/{}, output empty vcf file"
.format(input_dir, vcf_fn_prefix)))
compress_index_vcf(output_fn)
print_calling_step(output_fn=output_fn)
return
if vcf_fn_suffix is not None:
all_files = [
item for item in all_files if item.endswith(vcf_fn_suffix)
]
if len(all_files) == 0:
output_header(output_fn=output_fn,
reference_file_path=ref_fn,
sample_name=sample_name)
print(
log_warning(
"[WARNING] No vcf file found with suffix:{}/{}, output empty vcf file"
.format(input_dir, vcf_fn_prefix)))
compress_index_vcf(output_fn)
print_calling_step(output_fn=output_fn)
return
all_contigs_list = []
if contigs_fn and os.path.exists(contigs_fn):
with open(contigs_fn) as f:
all_contigs_list = [item.rstrip() for item in f.readlines()]
else:
exit(
log_error("[ERROR] Cannot find contig file {}. Exit!").format(
contigs_fn))
contigs_order = major_contigs_order + all_contigs_list
contigs_order_list = sorted(all_contigs_list,
key=lambda x: contigs_order.index(x))
row_count = 0
header = []
no_vcf_output = True
need_write_header = True
# only compress intermediate gvcf using lz4 output and keep final gvcf in bgzip format
output_bgzip_gvcf = vcf_fn_suffix == '.gvcf'
compress_gvcf = 'gvcf' in vcf_fn_suffix
if compress_gvcf:
lz4_path = subprocess.run("which lz4",
stdout=subprocess.PIPE,
shell=True).stdout.decode().rstrip()
compress_gvcf = True if lz4_path != "" else False
is_lz4_format = compress_gvcf
compress_gvcf_output = compress_gvcf and not output_bgzip_gvcf
if compress_gvcf_output:
write_fpo = open(output_fn, 'w')
write_proc = subprocess_popen(shlex.split("lz4 -c"),
stdin=subprocess.PIPE,
stdout=write_fpo,
stderr=subprocess.DEVNULL)
output = write_proc.stdin
else:
output = open(output_fn, 'w')
for contig in contigs_order_list:
contig_dict = defaultdict(str)
contig_vcf_fns = [fn for fn in all_files if contig in fn]
for vcf_fn in contig_vcf_fns:
file = os.path.join(input_dir, vcf_fn)
if is_lz4_format:
read_proc = subprocess_popen(shlex.split("{} {}".format(
"lz4 -fdc", file)),
stderr=subprocess.DEVNULL)
fn = read_proc.stdout
else:
fn = open(file, 'r')
for row in fn:
row_count += 1
if row[0] == '#':
# skip phasing command line only occur with --enable_phasing, otherwise would lead to hap.py evaluation failure
if row.startswith('##commandline='):
continue
if row not in header:
header.append(row)
continue
# use the first vcf header
columns = row.strip().split(maxsplit=3)
ctg_name, pos = columns[0], columns[1]
# skip vcf file sharing same contig prefix, ie, chr1 and chr11
if ctg_name != contig:
break
contig_dict[int(pos)] = row
no_vcf_output = False
fn.close()
if is_lz4_format:
read_proc.wait()
if need_write_header and len(header):
if output_bgzip_gvcf:
header = check_header_in_gvcf(header=header,
contigs_list=all_contigs_list)
output.write(''.join(header))
need_write_header = False
all_pos = sorted(contig_dict.keys())
for pos in all_pos:
output.write(contig_dict[pos])
if compress_gvcf_output:
write_proc.stdin.close()
write_proc.wait()
write_fpo.close()
return
else:
output.close()
if row_count == 0:
print(
log_warning("[WARNING] No vcf file found, output empty vcf file"))
output_header(output_fn=output_fn,
reference_file_path=ref_fn,
sample_name=sample_name)
compress_index_vcf(output_fn)
print_calling_step(output_fn=output_fn)
return
if no_vcf_output:
output_header(output_fn=output_fn,
reference_file_path=ref_fn,
sample_name=sample_name)
print(log_warning("[WARNING] No variant found, output empty vcf file"))
compress_index_vcf(output_fn)
print_calling_step(output_fn=output_fn)
return
if vcf_fn_suffix == ".tmp.gvcf":
return
if vcf_fn_suffix == ".gvcf":
print("[INFO] Need some time to compress and index GVCF file...")
compress_index_vcf(output_fn)
def get_training_array(tensor_fn,
var_fn,
bed_fn,
shuffle=True,
is_allow_duplicate_chr_pos=False):
tree = bed_tree_from(bed_file_path=bed_fn)
is_tree_empty = len(tree.keys()) == 0
Y = variant_map_from(var_fn, tree, is_tree_empty)
X = {}
f = subprocess_popen(shlex.split("gzip -fdc %s" % (tensor_fn)))
total = 0
mat = np.empty(input_tensor_size, dtype=np.float32)
for row in f.stdout:
chrom, coord, seq, mat = unpack_a_tensor_record(*(row.split()))
if not (is_tree_empty or is_region_in(tree, chrom, int(coord))):
continue
seq = seq.upper()
if seq[param.flankingBaseNum] not in BASIC_BASES:
continue
key = chrom + ":" + coord
x = np.reshape(mat, (no_of_positions, matrix_row, matrix_num))
for i in range(1, matrix_num):
x[:, :, i] -= x[:, :, 0]
if key not in X:
X[key] = np.copy(x)
elif is_allow_duplicate_chr_pos:
new_key = ""
for character in PREFIX_CHAR_STR:
tmp_key = character + key
if tmp_key not in X:
new_key = tmp_key
break
if len(new_key) > 0:
X[new_key] = np.copy(x)
is_reference = key not in Y
if is_reference:
Y[key] = output_labels_from_reference(
BASE2ACGT[seq[param.flankingBaseNum]])
total += 1
if total % 100000 == 0:
print("Processed %d tensors" % total, file=sys.stderr)
f.stdout.close()
f.wait()
# print "[INFO] size of X: {}, size of Y: {}".format(len(X), len(Y))
all_chr_pos = sorted(X.keys())
if shuffle == True:
np.random.shuffle(all_chr_pos)
X_compressed, Y_compressed, pos_compressed = [], [], []
X_array, Y_array, pos_array = [], [], []
count = 0
total = 0
for key in all_chr_pos:
total += 1
X_array.append(X[key])
del X[key]
if key in Y:
Y_array.append(Y[key])
pos_array.append(key)
if not is_allow_duplicate_chr_pos:
del Y[key]
elif is_allow_duplicate_chr_pos:
tmp_key = key[1:]
Y_array.append(Y[tmp_key])
pos_array.append(tmp_key)
count += 1
if count == param.bloscBlockSize:
X_compressed.append(blosc_pack_array(np.array(X_array)))
Y_compressed.append(blosc_pack_array(np.array(Y_array)))
pos_compressed.append(blosc_pack_array(np.array(pos_array)))
X_array, Y_array, pos_array = [], [], []
count = 0
if total % 50000 == 0:
print("Compressed %d/%d tensor" % (total, len(all_chr_pos)),
file=sys.stderr)
if count > 0:
X_compressed.append(blosc_pack_array(np.array(X_array)))
Y_compressed.append(blosc_pack_array(np.array(Y_array)))
pos_compressed.append(blosc_pack_array(np.array(pos_array)))
return total, X_compressed, Y_compressed, pos_compressed
