def test_CreateHdpTrainingData(self):
with tempfile.TemporaryDirectory() as tempdir:
# create fast5 dir
test_fast5 = os.path.join(tempdir, "test.fast5")
copyfile(self.fast5_paths[0], test_fast5)
# create fofn
test_out = os.path.join(tempdir, "test.hdp.tsv")
test_args = create_sa_sample_args(
fast5_dirs=[tempdir],
name="some_name",
fw_reference=self.ecoli_reference,
bwa_reference=self.ecoli_reference,
number_of_kmer_assignments=1,
probability_threshold=0,
kmers_from_reference=False)
working_folder = FolderHandler()
working_folder.open_folder(os.path.join(tempdir, "test_dir"))
sample = SignalAlignSample(working_folder=working_folder,
**test_args)
sample.analysis_files = [
self.assignment_file, self.assignment_file
]
out_path = CreateHdpTrainingData(
[sample],
test_out,
template=True,
complement=False,
verbose=False).write_hdp_training_file()
n_lines = count_lines_in_file(out_path)
self.assertEqual(n_lines, 3182)
with open(out_path, 'r') as fh1, open(self.test_hdp_training_data,
'r') as fh2:
self.assertEqual(sorted(list(fh1)), sorted(list(fh2)))
def test_processReferenceFasta_positions(self):
with tempfile.TemporaryDirectory() as tempdir:
work_folder = FolderHandler()
work_folder.open_folder(os.path.join(tempdir, "test_outdir"))
forward_ref, backward_ref = processReferenceFasta(
self.reference,
work_folder,
motifs=None,
positions_file=self.ambiguity_positions_file)
title, comment, seq = read_fasta(forward_ref).__next__()
self.assertEqual(
seq,
"ABTATTGAAGCATTTATCAGGGTTATTGTCTCATGAGCGGATACATATTTGAATGTATTTAGAAAAATAAACAAATAGGGGTTCCGCGCACATTTCCCCGAAAAGTGCCACCTGACGTCTAAGAAACCATTATTATCATGACATTAACCTATAAAAATAGGCGTATCACGAGGCCCTTTCGTCTCGCGCGTTTCGGTGATGACGGTGAAAACCTCTGACACATGCAGCTCCCGGAGACGGTCACAGCTTGTCTGTAAGCGGATGCCGGGAGCAGACAAGCCCGTCAGGGCGCGTCAGCGGGTGTTGGCGGGTGTCGGGGCTGGCTTAACTATGCGGCATCAGAGCAGATTGTACTGAGAGTGCACCATATGCGGTGTGAAATACCGCACAGATGCGTAAGGAGAAAATACCGCATCAGGCGCCATTCGCCATTCAGGCTGCGCAACTGTTGGGAAGGGCGATCGGTGCGGGCCTCTTCGCTATTACGCCAGCTGGCGAAAGGGGGATGTGCTGCAAGGCGATTAAGTTGGGTAACGCCAGGGTTTTCCCAGTCACGACGTTGTAAAACGACGGCCAGTGAATTCGAGCTCGGTACCCGGGGATCCTCTAGAGTCGACCTGCAGGCATGCAAGCTTGGCGTAATCATGGTCATAGCTGTTTCCTGTGTGAAATTGTTATCCGCTCACAATTCCACACAACATACGAGCCGGAAGCATAAAGTGTAAAGCCTGGGGTGCCTAATGAGTGAGCTAACTCACATTAATTGCGTTGCGCTCACTGCCCGCTTTCCAGTCGGGAAACCTGTCGTGCCAGCTGCATTAATGAATCGGCCAACGCGCGGGGAGAGGCGGTTTGCGTATTGGGCGCTCTTCCGCTTCCTCGCTCACTGACTCGCTGCGCTCGGTCGTTCGGCTGCGGCGAGCGGTATCAGCTCACTCAAAGGCGGTAATACGGTTATCCACAGAATCAGGGGATAACGCAGGAAAGAACATGTGAGCAAAAGGCCAGCAAAAGGCCAGGAACCGTAAAAAGGCCGCGTTGCTGGCGTTTTTCCATAGGCTCCGCCCCCCTGACGAGCATCACAAAAATCGACGCTCAAGTCAGAGGTGGCGAAACCCGACAGGACTATAAAGATACCAGGCGTTTCCCCCTGGAAGCTCCCTCGTGCGCTCTCCTGTTCCGACCCTGCCGCTTACCGGATACCTGTCCGCCTTTCTCCCTTCGGGAAGCGTGGCGCTTTCTCATAGCTCACGCTGTAGGTATCTCAGTTCGGTGTAGGTCGTTCGCTCCAAGCTGGGCTGTGTGCACGAACCCCCCGTTCAGCCCGACCGCTGCGCCTTATCCGGTAACTATCGTCTTGAGTCCAACCCGGTAAGACACGACTTATCGCCACTGGCAGCAGCCACTGGTAACAGGATTAGCAGAGCGAGGTATGTAGGCGGTGCTACAGAGTTCTTGAAGTGGTGGCCTAACTACGGCTACACTAGAAGAACAGTATTTGGTATCTGCGCTCTGCTGAAGCCAGTTACCTTCGGAAAAAGAGTTGGTAGCTCTTGATCCGGCAAACAAACCACCGCTGGTAGCGGTGGTTTTTTTGTTTGCAAGCAGCAGATTACGCGCAGAAAAAAAGGATCTCAAGAAGATCCTTTGATCTTTTCTACGGGGTCTGACGCTCAGTGGAACGAAAACTCACGTTAAGGGATTTTGGTCATGAGATTATCAAAAAGGATCTTCACCTAGATCCTTTTAAATTAAAAATGAAGTTTTAAATCAATCTAAAGTATATATGAGTAAACTTGGTCTGACAGTTACCAATGCTTAATCAGTGAGGCACCTATCTCAGCGATCTGTCTATTTCGTTCATCCATAGTTGCCTGACTCCCCGTCGTGTAGATAACTACGATACGGGAGGGCTTACCATCTGGCCCCAGTGCTGCAATGATACCGCGAGACCCACGCTCACCGGCTCCAGATTTATCAGCAATAAACCAGCCAGCCGGAAGGGCCGAGCGCAGAAGTGGTCCTGCAACTTTATCCGCCTCCATCCAGTCTATTAATTGTTGCCGGGAAGCTAGAGTAAGTAGTTCGCCAGTTAATAGTTTGCGCAACGTTGTTGCCATTGCTACAGGCATCGTGGTGTCACGCTCGTCGTTTGGTATGGCTTCATTCAGCTCCGGTTCCCAACGATCAAGGCGAGTTACATGATCCCCCATGTTGTGCAAAAAAGCGGTTAGCTCCTTCGGTCCTCCGATCGTTGTCAGAAGTAAGTTGGCCGCAGTGTTATCACTCATGGTTATGGCAGCACTGCATAATTCTCTTACTGTCATGCCATCCGTAAGATGCTTTTCTGTGACTGGTGAGTACTCAACCAAGTCATTCTGAGAATAGTGTATGCGGCGACCGAGTTGCTCTTGCCCGGCGTCAATACGGGATAATACCGCGCCACATAGCAGAACTTTAAAAGTGCTCATCATTGGAAAACGTTCTTCGGGGCGAAAACTCTCAAGGATCTTACCGCTGTTGAGATCCAGTTCGATGTAACCCACTCGTGCACCCAACTGATCTTCAGCATCTTTTACTTTCACCAGCGTTTCTGGGTGAGCAAAAACAGGAAGGCAAAATGCCGCAAAAAAGGGAATAAGGGCGACACGGAAATGTTGAATACTCATACTCTTCCTTTTTCAAT"
)
title, comment, seq = read_fasta(backward_ref).__next__()
self.assertEqual(
seq,
"TBATAACTTCGTAAATAGTCCCAATAACAGAGTACTCGCCTATGTATAAACTTACATAAATCTTTTTATTTGTTTATCCCCAAGGCGCGTGTAAAGGGGCTTTTCACGGTGGACTGCAGATTCTTTGGTAATAATAGTACTGTAATTGGATATTTTTATCCGCATAGTGCTCCGGGAAAGCAGAGCGCGCAAAGCCACTACTGCCACTTTTGGAGACTGTGTACGTCGAGGGCCTCTGCCAGTGTCGAACAGACATTCGCCTACGGCCCTCGTCTGTTCGGGCAGTCCCGCGCAGTCGCCCACAACCGCCCACAGCCCCGACCGAATTGATACGCCGTAGTCTCGTCTAACATGACTCTCACGTGGTATACGCCACACTTTATGGCGTGTCTACGCATTCCTCTTTTATGGCGTAGTCCGCGGTAAGCGGTAAGTCCGACGCGTTGACAACCCTTCCCGCTAGCCACGCCCGGAGAAGCGATAATGCGGTCGACCGCTTTCCCCCTACACGACGTTCCGCTAATTCAACCCATTGCGGTCCCAAAAGGGTCAGTGCTGCAACATTTTGCTGCCGGTCACTTAAGCTCGAGCCATGGGCCCCTAGGAGATCTCAGCTGGACGTCCGTACGTTCGAACCGCATTAGTACCAGTATCGACAAAGGACACACTTTAACAATAGGCGAGTGTTAAGGTGTGTTGTATGCTCGGCCTTCGTATTTCACATTTCGGACCCCACGGATTACTCACTCGATTGAGTGTAATTAACGCAACGCGAGTGACGGGCGAAAGGTCAGCCCTTTGGACAGCACGGTCGACGTAATTACTTAGCCGGTTGCGCGCCCCTCTCCGCCAAACGCATAACCCGCGAGAAGGCGAAGGAGCGAGTGACTGAGCGACGCGAGCCAGCAAGCCGACGCCGCTCGCCATAGTCGAGTGAGTTTCCGCCATTATGCCAATAGGTGTCTTAGTCCCCTATTGCGTCCTTTCTTGTACACTCGTTTTCCGGTCGTTTTCCGGTCCTTGGCATTTTTCCGGCGCAACGACCGCAAAAAGGTATCCGAGGCGGGGGGACTGCTCGTAGTGTTTTTAGCTGCGAGTTCAGTCTCCACCGCTTTGGGCTGTCCTGATATTTCTATGGTCCGCAAAGGGGGACCTTCGAGGGAGCACGCGAGAGGACAAGGCTGGGACGGCGAATGGCCTATGGACAGGCGGAAAGAGGGAAGCCCTTCGCACCGCGAAAGAGTATCGAGTGCGACATCCATAGAGTCAAGCCACATCCAGCAAGCGAGGTTCGACCCGACACACGTGCTTGGGGGGCAAGTCGGGCTGGCGACGCGGAATAGGCCATTGATAGCAGAACTCAGGTTGGGCCATTCTGTGCTGAATAGCGGTGACCGTCGTCGGTGACCATTGTCCTAATCGTCTCGCTCCATACATCCGCCACGATGTCTCAAGAACTTCACCACCGGATTGATGCCGATGTGATCTTCTTGTCATAAACCATAGACGCGAGACGACTTCGGTCAATGGAAGCCTTTTTCTCAACCATCGAGAACTAGGCCGTTTGTTTGGTGGCGACCATCGCCACCAAAAAAACAAACGTTCGTCGTCTAATGCGCGTCTTTTTTTCCTAGAGTTCTTCTAGGAAACTAGAAAAGATGCCCCAGACTGCGAGTCACCTTGCTTTTGAGTGCAATTCCCTAAAACCAGTACTCTAATAGTTTTTCCTAGAAGTGGATCTAGGAAAATTTAATTTTTACTTCAAAATTTAGTTAGATTTCATATATACTCATTTGAACCAGACTGTCAATGGTTACGAATTAGTCACTCCGTGGATAGAGTCGCTAGACAGATAAAGCAAGTAGGTATCAACGGACTGAGGGGCAGCACATCTATTGATGCTATGCCCTCCCGAATGGTAGACCGGGGTCACGACGTTACTATGGCGCTCTGGGTGCGAGTGGCCGAGGTCTAAATAGTCGTTATTTGGTCGGTCGGCCTTCCCGGCTCGCGTCTTCACCAGGACGTTGAAATAGGCGGAGGTAGGTCAGATAATTAACAACGGCCCTTCGATCTCATTCATCAAGCGGTCAATTATCAAACGCGTTGCAACAACGGTAACGATGTCCGTAGCACCACAGTGCGAGCAGCAAACCATACCGAAGTAAGTCGAGGCCAAGGGTTGCTAGTTCCGCTCAATGTACTAGGGGGTACAACACGTTTTTTCGCCAATCGAGGAAGCCAGGAGGCTAGCAACAGTCTTCATTCAACCGGCGTCACAATAGTGAGTACCAATACCGTCGTGACGTATTAAGAGAATGACAGTACGGTAGGCATTCTACGAAAAGACACTGACCACTCATGAGTTGGTTCAGTAAGACTCTTATCACATACGCCGCTGGCTCAACGAGAACGGGCCGCAGTTATGCCCTATTATGGCGCGGTGTATCGTCTTGAAATTTTCACGAGTAGTAACCTTTTGCAAGAAGCCCCGCTTTTGAGAGTTCCTAGAATGGCGACAACTCTAGGTCAAGCTACATTGGGTGAGCACGTGGGTTGACTAGAAGTCGTAGAAAATGAAAGTGGTCGCAAAGACCCACTCGTTTTTGTCCTTCCGTTTTACGGCGTTTTTTCCCTTATTCCCGCTGTGCCTTTACAACTTATGAGTATGAGAAGGAAAAAGTTA"
)
forward_ref, backward_ref = processReferenceFasta(
self.reference, work_folder, motifs=None, positions_file=None)
title, comment, seq = read_fasta(forward_ref).__next__()
self.assertEqual(
seq,
"ATTATTGAAGCATTTATCAGGGTTATTGTCTCATGAGCGGATACATATTTGAATGTATTTAGAAAAATAAACAAATAGGGGTTCCGCGCACATTTCCCCGAAAAGTGCCACCTGACGTCTAAGAAACCATTATTATCATGACATTAACCTATAAAAATAGGCGTATCACGAGGCCCTTTCGTCTCGCGCGTTTCGGTGATGACGGTGAAAACCTCTGACACATGCAGCTCCCGGAGACGGTCACAGCTTGTCTGTAAGCGGATGCCGGGAGCAGACAAGCCCGTCAGGGCGCGTCAGCGGGTGTTGGCGGGTGTCGGGGCTGGCTTAACTATGCGGCATCAGAGCAGATTGTACTGAGAGTGCACCATATGCGGTGTGAAATACCGCACAGATGCGTAAGGAGAAAATACCGCATCAGGCGCCATTCGCCATTCAGGCTGCGCAACTGTTGGGAAGGGCGATCGGTGCGGGCCTCTTCGCTATTACGCCAGCTGGCGAAAGGGGGATGTGCTGCAAGGCGATTAAGTTGGGTAACGCCAGGGTTTTCCCAGTCACGACGTTGTAAAACGACGGCCAGTGAATTCGAGCTCGGTACCCGGGGATCCTCTAGAGTCGACCTGCAGGCATGCAAGCTTGGCGTAATCATGGTCATAGCTGTTTCCTGTGTGAAATTGTTATCCGCTCACAATTCCACACAACATACGAGCCGGAAGCATAAAGTGTAAAGCCTGGGGTGCCTAATGAGTGAGCTAACTCACATTAATTGCGTTGCGCTCACTGCCCGCTTTCCAGTCGGGAAACCTGTCGTGCCAGCTGCATTAATGAATCGGCCAACGCGCGGGGAGAGGCGGTTTGCGTATTGGGCGCTCTTCCGCTTCCTCGCTCACTGACTCGCTGCGCTCGGTCGTTCGGCTGCGGCGAGCGGTATCAGCTCACTCAAAGGCGGTAATACGGTTATCCACAGAATCAGGGGATAACGCAGGAAAGAACATGTGAGCAAAAGGCCAGCAAAAGGCCAGGAACCGTAAAAAGGCCGCGTTGCTGGCGTTTTTCCATAGGCTCCGCCCCCCTGACGAGCATCACAAAAATCGACGCTCAAGTCAGAGGTGGCGAAACCCGACAGGACTATAAAGATACCAGGCGTTTCCCCCTGGAAGCTCCCTCGTGCGCTCTCCTGTTCCGACCCTGCCGCTTACCGGATACCTGTCCGCCTTTCTCCCTTCGGGAAGCGTGGCGCTTTCTCATAGCTCACGCTGTAGGTATCTCAGTTCGGTGTAGGTCGTTCGCTCCAAGCTGGGCTGTGTGCACGAACCCCCCGTTCAGCCCGACCGCTGCGCCTTATCCGGTAACTATCGTCTTGAGTCCAACCCGGTAAGACACGACTTATCGCCACTGGCAGCAGCCACTGGTAACAGGATTAGCAGAGCGAGGTATGTAGGCGGTGCTACAGAGTTCTTGAAGTGGTGGCCTAACTACGGCTACACTAGAAGAACAGTATTTGGTATCTGCGCTCTGCTGAAGCCAGTTACCTTCGGAAAAAGAGTTGGTAGCTCTTGATCCGGCAAACAAACCACCGCTGGTAGCGGTGGTTTTTTTGTTTGCAAGCAGCAGATTACGCGCAGAAAAAAAGGATCTCAAGAAGATCCTTTGATCTTTTCTACGGGGTCTGACGCTCAGTGGAACGAAAACTCACGTTAAGGGATTTTGGTCATGAGATTATCAAAAAGGATCTTCACCTAGATCCTTTTAAATTAAAAATGAAGTTTTAAATCAATCTAAAGTATATATGAGTAAACTTGGTCTGACAGTTACCAATGCTTAATCAGTGAGGCACCTATCTCAGCGATCTGTCTATTTCGTTCATCCATAGTTGCCTGACTCCCCGTCGTGTAGATAACTACGATACGGGAGGGCTTACCATCTGGCCCCAGTGCTGCAATGATACCGCGAGACCCACGCTCACCGGCTCCAGATTTATCAGCAATAAACCAGCCAGCCGGAAGGGCCGAGCGCAGAAGTGGTCCTGCAACTTTATCCGCCTCCATCCAGTCTATTAATTGTTGCCGGGAAGCTAGAGTAAGTAGTTCGCCAGTTAATAGTTTGCGCAACGTTGTTGCCATTGCTACAGGCATCGTGGTGTCACGCTCGTCGTTTGGTATGGCTTCATTCAGCTCCGGTTCCCAACGATCAAGGCGAGTTACATGATCCCCCATGTTGTGCAAAAAAGCGGTTAGCTCCTTCGGTCCTCCGATCGTTGTCAGAAGTAAGTTGGCCGCAGTGTTATCACTCATGGTTATGGCAGCACTGCATAATTCTCTTACTGTCATGCCATCCGTAAGATGCTTTTCTGTGACTGGTGAGTACTCAACCAAGTCATTCTGAGAATAGTGTATGCGGCGACCGAGTTGCTCTTGCCCGGCGTCAATACGGGATAATACCGCGCCACATAGCAGAACTTTAAAAGTGCTCATCATTGGAAAACGTTCTTCGGGGCGAAAACTCTCAAGGATCTTACCGCTGTTGAGATCCAGTTCGATGTAACCCACTCGTGCACCCAACTGATCTTCAGCATCTTTTACTTTCACCAGCGTTTCTGGGTGAGCAAAAACAGGAAGGCAAAATGCCGCAAAAAAGGGAATAAGGGCGACACGGAAATGTTGAATACTCATACTCTTCCTTTTTCAAT"
)
self.assertIsNone(backward_ref)
self.assertEqual(forward_ref, self.reference)
self.assertRaises(RuntimeError,
processReferenceFasta,
self.reference,
work_folder,
motifs="something",
positions_file=self.ambiguity_positions_file)
def test_trim_num_files_in_sample(self):
with tempfile.TemporaryDirectory() as tempdir:
working_folder = FolderHandler()
working_folder.open_folder(os.path.join(tempdir, "test_dir"))
test_args = create_sa_sample_args(
fast5_dirs=[self.fast5_dir],
name="some_name",
fw_reference=self.ecoli_reference)
sample = SignalAlignSample(working_folder=working_folder,
**test_args)
n_bases = 10000
fast5_files = trim_num_files_in_sample(sample,
n_bases,
False,
verbose=False)
bases = 0
for fast5_file in fast5_files:
bases += get_1d_length(fast5_file)
self.assertLessEqual(bases, n_bases)
fast5_files = trim_num_files_in_sample(sample,
n_bases,
True,
verbose=False)
bases = 0
for fast5_file in fast5_files:
bases += get_2d_length(fast5_file)
self.assertLessEqual(bases, n_bases)
self.assertRaises(AssertionError,
trim_num_files_in_sample,
sample,
1,
False,
verbose=False)
def __init__(self,
in_fast5,
destination,
stateMachineType,
bwa_index,
in_templateHmm,
in_complementHmm,
in_templateHdp,
in_complementHdp,
threshold,
diagonal_expansion,
constraint_trim,
degenerate,
twoD_chemistry,
forward_reference,
backward_reference=None,
target_regions=None,
output_format="full",
embed=False,
event_table=False):
self.in_fast5 = in_fast5 # fast5 file to align
self.destination = destination # place where the alignments go, should already exist
self.stateMachineType = stateMachineType # flag for signalMachine
self.bwa_index = bwa_index # index of reference sequence
self.threshold = threshold # min posterior probability to keep
self.diagonal_expansion = diagonal_expansion # alignment algorithm param
self.constraint_trim = constraint_trim # alignment algorithm param
self.output_format = output_format # smaller output files
self.degenerate = degenerate # set of nucleotides for degenerate characters
self.twoD_chemistry = twoD_chemistry # flag for 2D sequencing runs
self.temp_folder = FolderHandler(
) # object for holding temporary files (non-toil)
self.read_name = self.in_fast5.split(
"/")[-1][:-6] # get the name without the '.fast5'
self.target_regions = target_regions
self.output_formats = {"full": 0, "variantCaller": 1, "assignments": 2}
self.embed = embed # embed the output into the fast5 file
self.event_table = event_table # specify which event table to use to generate alignments
self.backward_reference = backward_reference # fasta path to backward reference if modified bases are used
self.forward_reference = forward_reference # fasta path to forward reference
if (in_templateHmm is not None) and os.path.isfile(in_templateHmm):
self.in_templateHmm = in_templateHmm
else:
self.in_templateHmm = None
if (in_complementHmm is not None) and os.path.isfile(in_complementHmm):
self.in_complementHmm = in_complementHmm
else:
self.in_complementHmm = None
# similarly for HDPs
if (in_templateHdp is not None) and os.path.isfile(in_templateHdp):
self.in_templateHdp = in_templateHdp
else:
self.in_templateHdp = None
if (in_complementHdp is not None) and os.path.isfile(in_complementHdp):
self.in_complementHdp = in_complementHdp
else:
self.in_complementHdp = None
def test_mea_alignment_close_to_guide(self):
from signalalign.validateSignalAlignment import get_all_event_summaries, ABS_SA_ALIGNMENT_DIFF, MEA
from signalalign.utils.fileHandlers import FolderHandler
from signalalign.signalAlignment import create_signalAlignment_args
import shutil
import tempfile
import glob
ecoli_reference = os.path.join(MeaTest.HOME, "tests/test_sequences/E.coli_K12.fasta")
fast5_dir = os.path.join(MeaTest.HOME, "tests/minion_test_reads/1D")
template_hmm = os.path.join(MeaTest.HOME, "models/testModelR9_acgt_template.model")
path_to_bin = os.path.join(MeaTest.HOME, 'bin')
threshold = 11
# make directory to put temporary files and output location
output_root = tempfile.TemporaryDirectory()
temp_root = FolderHandler()
temp_fast5_dir = temp_root.open_folder(os.path.join(output_root.name, "temp_fast5"))
temp_signal_align_dir = os.path.join(output_root.name, "temp_signalAlign")
if os.path.isdir(temp_signal_align_dir):
shutil.rmtree(temp_signal_align_dir)
assert not os.path.isdir(temp_signal_align_dir)
temp_signal_align = temp_root.open_folder(temp_signal_align_dir)
# get input files
orig_fast5s = glob.glob(os.path.join(fast5_dir, "*.fast5"))
self.assertTrue(len(orig_fast5s) > 0, "Incorrect fast5 location: {}".format(fast5_dir))
fast5s = list()
for file in orig_fast5s:
dest = os.path.join(temp_fast5_dir, os.path.basename(file))
shutil.copy(file, dest)
fast5s.append(dest)
# get alignment args
alignment_args = create_signalAlignment_args(bwa_reference=ecoli_reference,
in_templateHmm=template_hmm,
destination=temp_signal_align_dir,
forward_reference=ecoli_reference,
path_to_bin=path_to_bin,
constraint_trim=0,
traceBackDiagonals=100,
diagonal_expansion=0,
embed=True)
# get summaries
all_event_summaries = get_all_event_summaries(fast5s, alignment_args, aln_dist_threshold=threshold,
generate_plot=False, verbose=False)
for fast5 in all_event_summaries.keys():
f5_name = os.path.basename(fast5)
event_summaries = all_event_summaries[fast5]
max_mea_aln_diff = max(list(map(lambda x: x[ABS_SA_ALIGNMENT_DIFF],
list(filter(lambda x: x[MEA], event_summaries)))))
self.assertTrue(max_mea_aln_diff <= threshold,
"MEA produced alignment greater than {} positions from guide alignment for {}".format(
max_mea_aln_diff, f5_name))
def test_multithread_signal_alignment(self):
with tempfile.TemporaryDirectory() as tempdir:
working_folder = FolderHandler()
working_folder.open_folder(os.path.join(tempdir, "test_dir"))
# create signalalign args
assert os.path.isfile(self.template_hmm)
signal_align_arguments = create_signalAlignment_args(
bwa_reference=self.ecoli_reference,
in_templateHmm=self.template_hmm,
destination=working_folder.path,
forward_reference=self.ecoli_reference,
path_to_bin=self.path_to_bin)
fast5_files = self.fast5_paths[:1]
with captured_output() as (out, err):
output_files = multithread_signal_alignment(
signal_align_arguments,
fast5_files,
2,
forward_reference=self.ecoli_reference)
self.assertEqual(len(output_files), len(fast5_files))
# round 2
working_folder = FolderHandler()
working_folder.open_folder(os.path.join(tempdir, "test_dir2"))
# create signalalign args
assert os.path.isfile(self.template_hmm)
signal_align_arguments = create_signalAlignment_args(
bwa_reference=self.ecoli_reference,
in_templateHmm=self.template_hmm,
destination=working_folder.path,
forward_reference=self.ecoli_reference,
path_to_bin=self.path_to_bin)
filter_reads_generator = filter_reads_to_string_wrapper(
filter_reads(self.bam, self.readdb, [self.test_dir]))
output_files = multithread_signal_alignment(
signal_align_arguments, [],
2,
forward_reference=self.ecoli_reference,
filter_reads_to_string_wrapper=filter_reads_generator,
debug=True)
self.assertEqual(len(output_files), 3)
def test_embed_with_both(self):
signal_file_reads = os.path.join(self.HOME,
"tests/minion_test_reads/pUC/")
template_model = os.path.join(
self.HOME, "models/testModelR9_5mer_acegt_template.model")
complement_model = os.path.join(
self.HOME, "models/testModelR9_5mer_acegt_complement.model")
puc_reference = os.path.join(self.HOME,
"tests/test_sequences/pUC19_SspI.fa")
signal_file_guide_alignment = os.path.join(
self.HOME, "tests/minion_test_reads/pUC/puc.bam")
with tempfile.TemporaryDirectory() as tempdir:
new_dir = os.path.join(tempdir, "new_dir")
if os.path.exists(new_dir):
shutil.rmtree(new_dir)
working_folder = FolderHandler()
working_folder.open_folder(os.path.join(tempdir, "test_dir"))
shutil.copytree(signal_file_reads, new_dir)
args = create_signalAlignment_args(
alignment_file=signal_file_guide_alignment,
bwa_reference=puc_reference,
forward_reference=puc_reference,
in_templateHmm=template_model,
path_to_bin=self.path_to_bin,
destination=working_folder.path,
embed=True,
output_format="both",
filter_reads=0,
twoD_chemistry=True,
in_complementHmm=complement_model,
delete_tmp=True)
final_args = merge_dicts([
args,
dict(in_fast5=os.path.join(
new_dir,
"makeson_PC_20160807_FNFAD20242_MN17284_sequencing_run_MA_470_R9_pUC_g_PCR_BC_08_07_16_93165_ch1_read176_strand.fast5"
))
])
handle = SignalAlignment(**final_args)
handle.run()
f5fh = Fast5(
os.path.join(
new_dir,
"makeson_PC_20160807_FNFAD20242_MN17284_sequencing_run_MA_470_R9_pUC_g_PCR_BC_08_07_16_93165_ch1_read176_strand.fast5"
))
mea = f5fh.get_signalalign_events(mea=True)
sam = f5fh.get_signalalign_events(sam=True)
self.assertEqual(mea[0]["raw_start"], 2879)
self.assertEqual(sam[0], "0")
self.assertEqual(len(os.listdir(working_folder.path)), 2)
def test_processReferenceFasta_positions(self):
with tempfile.TemporaryDirectory() as tempdir:
work_folder = FolderHandler()
work_folder.open_folder(os.path.join(tempdir, "test_outdir"))
forward_ref, backward_ref = processReferenceFasta(self.reference, work_folder, motifs=[["AC", "EC"]],
positions_file=None, name="")
title, comment, seq = read_fasta(forward_ref).__next__()
self.assertEqual(seq.find("AC"), -1)
self.assertEqual(seq.find("EC"), 42)
title, comment, seq = read_fasta(backward_ref).__next__()
self.assertEqual(seq.find("AC"), -1)
self.assertEqual(seq.find("EC"), 5)
def test_multithread_signal_alignment_samples(self):
with tempfile.TemporaryDirectory() as tempdir:
working_folder = FolderHandler()
test_fast5 = os.path.join(
tempdir,
"miten_PC_20160820_FNFAD20259_MN17223_mux_scan_AMS_158_R9_WGA_Ecoli_08_20_16_83098_ch138_read23_strand.fast5"
)
num_files = 1
copyfile(self.fast5_paths[0], test_fast5)
working_folder.open_folder(os.path.join(tempdir, "test_dir"))
# create signalalign args
signal_align_arguments = create_signalAlignment_args(
in_templateHmm=self.template_hmm,
destination=working_folder.path,
path_to_bin=self.path_to_bin)
# create samples
samples = []
options = create_sa_sample_args(fast5_dirs=[tempdir],
name="some_name",
fw_reference=self.ecoli_reference,
bwa_reference=self.ecoli_reference,
readdb=self.fast5_readdb,
alignment_file=self.fast5_bam)
samples.append(
SignalAlignSample(working_folder=working_folder, **options))
options["name"] = "some_name2"
samples.append(
SignalAlignSample(working_folder=working_folder, **options))
# with captured_output() as (out, err):
samples = multithread_signal_alignment_samples(
samples, signal_align_arguments, 2)
self.assertSetEqual(set([sample.name for sample in samples]),
{'some_name', 'some_name2'})
for sample in samples:
if sample.name == "some_name":
self.assertEqual(len(sample.analysis_files), num_files)
if sample.name == "some_name2":
self.assertEqual(len(sample.analysis_files), num_files)
for file_path in sample.analysis_files:
self.assertTrue(os.path.isfile(file_path))
options["name"] = "some_name"
samples.append(
SignalAlignSample(working_folder=working_folder, **options))
self.assertRaises(AssertionError,
multithread_signal_alignment_samples, samples,
signal_align_arguments, 2)
def test_variant_calling_with_multiple_paths_rna(self):
with tempfile.TemporaryDirectory() as tempdir:
new_dir = os.path.join(tempdir, "new_dir")
if os.path.exists(new_dir):
shutil.rmtree(new_dir)
working_folder = FolderHandler()
working_folder.open_folder(os.path.join(tempdir, "test_dir"))
shutil.copytree(self.test_dir_rna, new_dir)
args = create_signalAlignment_args(
alignment_file=self.rna_bam,
bwa_reference=self.rna_reference,
forward_reference=os.path.join(
self.HOME,
"tests/test_sequences/fake_rna_replace/forward.fake_rna_atg.fake_rna_ref.fa"
),
backward_reference=os.path.join(
self.HOME,
"tests/test_sequences/fake_rna_replace/backward.fake_rna_atg.fake_rna_ref.fa"
),
in_templateHmm=os.path.join(
self.HOME,
"models/fake_testModelR9p4_5mer_acfgt_RNA.model"),
path_to_bin=self.path_to_bin,
destination=working_folder.path,
embed=False,
output_format="full",
filter_reads=0,
twoD_chemistry=False,
delete_tmp=True,
degenerate="m6a",
check_for_temp_file_existance=False)
multithread_signal_alignment(args,
list_dir(new_dir, ext="fast5"),
worker_count=8,
forward_reference=None,
debug=True,
filter_reads_to_string_wrapper=None)
self.assertEqual(len(os.listdir(working_folder.path)), 2)
def __init__(self, args):
# TODO Need to create docs here
"""Initialize all objects the training routine may need"""
# executable
self.buildHdpUtil = None
# HDP type
self.int_hdp_type = None
# load json and create dot dictionary of all the parameters
self.args = args
# check output directory
self.args.output_dir = os.path.abspath(self.args.output_dir)
assert os.path.exists(self.args.output_dir), "Output directory does not exist. " \
"output_dir: {}".format(self.args.output_dir)
self.working_folder = FolderHandler()
self.working_path = self.working_folder.open_folder(
os.path.join(self.args.output_dir, "tempFiles_trainModels"))
# create samples from self.args.samples
self.samples = self._create_samples()
# Current model paths
self.template_hmm_model_path = self.args.template_hmm_model
self.template_hdp_model_path = self.args.template_hdp_model
self.complement_hmm_model_path = self.args.complement_hmm_model
self.complement_hdp_model_path = self.args.complement_hdp_model
# Current SignalHmm model objects
self.complement_model = None
self.template_model = None
# globals for experiments
self.path_to_bin = self.args.path_to_bin
self.debug = self.args.debug
self.two_d = self.args.two_d
self.job_count = self.args.job_count
# state machine type changes for SignalAlignment so it can expect an HDP or not
self.state_machine_type = "threeState"
self.kmer_length = None
self.alphabet = None
# check config file
self._check_config()
def test_SignalAlignSample(self):
with tempfile.TemporaryDirectory() as tempdir:
# create fast5 dir
test_fast5 = os.path.join(tempdir, "test.fast5")
copyfile(self.fast5_paths[0], test_fast5)
# create fofn
test_out = os.path.join(tempdir, "test.fofn")
with open(test_out, 'w+') as fofn_file:
print(test_fast5, file=fofn_file)
test_args = create_sa_sample_args(
fast5_dirs=[tempdir, tempdir],
name="some_name",
fofns=[test_out, test_out],
fw_reference=self.ecoli_reference,
bwa_reference=self.ecoli_reference)
working_folder = FolderHandler()
working_folder.open_folder(os.path.join(tempdir, "test_dir"))
sample = SignalAlignSample(working_folder=working_folder,
**test_args)
def estimate_params(fast5, binary_path="./estimateNanoporeParams",
template_lookup_table="../models/testModelR9p4_acegt_template.model",
complement_lookup_table="../models/testModelR9_complement_pop2.model",
twoD=False, verbose=False):
temp_folder = FolderHandler()
temp_folder.open_folder("npParamEstimation")
read_name = fast5.split("/")[-1][:-6] # get the name without the '.fast5'
npRead_path = temp_folder.add_file_path(read_name + ".npRead")
npRead_fasta = temp_folder.add_file_path(read_name + ".seq.fasta")
if twoD:
success, version, complement = get_npRead_2dseq_and_models(fast5=fast5,
npRead_path=npRead_path,
twod_read_path=npRead_fasta)
# print(version, complement)
else:
success, version, complement = prepareOneD(fast5=fast5, npRead_path=npRead_path, oneD_read_path=npRead_fasta)
# print(version, complement)
if success is False:
return False
command = "{bin} -T {tLuT} -C {cLuT} -q {npRead}" \
"".format(bin=binary_path, tLuT=template_lookup_table, cLuT=complement_lookup_table, npRead=npRead_path)
if verbose:
print("running command {command}".format(command=command), file=sys.stderr)
# os.system(command)
result = subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT)
params = result.split()
param_dict = dict(list(zip([bytes.decode(x) for x in params[::2]], [float(x) for x in params[1::2]])))
# print(type(param_dict["scale"]))
# clean up temp folder
temp_folder.remove_file(npRead_path)
temp_folder.remove_file(npRead_fasta)
temp_folder.remove_folder()
return param_dict
def test_signal_file_and_alignment(self):
signal_file_reads = os.path.join(
self.HOME, "tests/minion_test_reads/no_event_data_1D_ecoli")
template_model = os.path.join(
self.HOME, "models/testModelR9p4_5mer_acegt_template.model")
ecoli_reference = os.path.join(
self.HOME, "tests/test_sequences/E.coli_K12.fasta")
signal_file_guide_alignment = os.path.join(
self.HOME, "tests/minion_test_reads/oneD_alignments.sam")
with tempfile.TemporaryDirectory() as tempdir:
new_dir = os.path.join(tempdir, "new_dir")
working_folder = FolderHandler()
working_folder.open_folder(os.path.join(tempdir, "test_dir"))
shutil.copytree(signal_file_reads, new_dir)
args = create_signalAlignment_args(
alignment_file=signal_file_guide_alignment,
bwa_reference=ecoli_reference,
forward_reference=ecoli_reference,
in_templateHmm=template_model,
path_to_bin=self.path_to_bin,
destination=working_folder.path)
final_args = merge_dicts([
args,
dict(in_fast5=os.path.join(
new_dir,
"LomanLabz_PC_20161025_FNFAB42699_MN17633_sequencing_run_20161025_E_coli_native_450bps_82361_ch6_read347_strand.fast5"
))
])
handle = SignalAlignment(**final_args)
handle.run()
self.assertEqual(len(os.listdir(working_folder.path)), 1)
self.assertEqual(
sorted(os.listdir(working_folder.path))[0],
"9e4d14b1-8167-44ef-9fdb-5c29dd0763fd.sm.backward.tsv")
def main(args):
args = parse_args()
# make directory to put temporary files
temp_folder = FolderHandler()
temp_dir_path = temp_folder.open_folder(args.out + "npParamEstimation")
fast5s = [x for x in os.listdir(args.files_dir) if x.endswith(".fast5")]
if len(fast5s) > args.nb_files:
shuffle(fast5s)
fast5s = fast5s[:args.nb_files]
for fast5 in fast5s:
print(fast5)
# estimate_params(fast5=args.files_dir + fast5, working_folder=temp_folder, bwa_index=bwa_ref_index,
# forward_reference_path=plus_strand_sequence, backward_reference_path=minus_strand_sequence,
# threshold=args.threshold)
try:
params = estimate_params(fast5=args.files_dir + fast5, twoD=True)
print(params)
except Exception as e:
print(e)
temp_folder.remove_folder()
return True
def main(args):
# parse args
args = parse_args()
command_line = " ".join(sys.argv[:])
print("Command Line: {cmdLine}\n".format(cmdLine=command_line),
file=sys.stderr)
start_message = """
# Starting Jamison Error-Correction
# Aligning files from: {fileDir}
# Aligning to reference: {reference}
# Aligning maximum of {nbFiles} files
# Using model: {model}
# Using banding: {banding}
# Aligning to regions in: {regions}
# Non-default template HMM: {inThmm}
# Non-default complement HMM: {inChmm}
# Template HDP: {tHdp}
# Complement HDP: {cHdp}
# Performing {cycles} cycles
""".format(fileDir=args.files_dir,
reference=args.ref,
nbFiles=args.nb_files,
banding=args.banded,
inThmm=args.in_T_Hmm,
inChmm=args.in_C_Hmm,
model=args.stateMachineType,
regions=args.target_regions,
tHdp=args.templateHDP,
cHdp=args.complementHDP,
cycles=args.cycles)
print(start_message, file=sys.stdout)
if not os.path.isfile(args.ref):
print("Did not find valid reference file", file=sys.stderr)
sys.exit(1)
temp_folder = FolderHandler()
temp_dir_path = temp_folder.open_folder(args.out +
"tempFiles_errorCorrection")
# initialize to input fasta
reference_sequence_path = args.ref
# list of alignment files
fast5s = cull_fast5_files(args.files_dir, args.nb_files)
for cycle in range(0, args.cycles):
# index the reference for bwa this is a string with the path to the index
bwa_ref_index = get_bwa_index(reference_sequence_path, temp_dir_path)
# unpack the reference sequence
reference_sequence_string = get_first_sequence(reference_sequence_path)
alignment_args = {
"path_to_EC_refs": None,
"destination": temp_dir_path,
"stateMachineType": args.stateMachineType,
"bwa_index": bwa_ref_index,
"in_templateHmm": args.in_T_Hmm,
"in_complementHmm": args.in_C_Hmm,
"in_templateHdp": args.templateHDP,
"in_complementHdp": args.complementHDP,
"banded": args.banded,
"sparse_output": True,
"threshold": args.threshold,
"diagonal_expansion": args.diag_expansion,
"constraint_trim": args.constraint_trim,
"target_regions": None,
"degenerate": degenerate_enum(args.degenerate),
}
proposals = scan_for_proposals(temp_folder, STEP,
reference_sequence_string, fast5s,
alignment_args, args.nb_jobs)
proposals = group_sites_in_window(proposals, 6)
print("Cycle {cycle} - Got {nb} sites to check: {sites}".format(
nb=len(proposals), sites=proposals, cycle=cycle))
updated_reference_string = update_reference_with_marginal_probs(
temp_folder, proposals, reference_sequence_string, fast5s,
alignment_args, args.nb_jobs)
updated_reference_path = temp_folder.add_file_path(
"cycle_snapshot.{cycle}.fa".format(cycle=cycle))
write_fasta("jamison{}".format(cycle), updated_reference_string,
open(updated_reference_path, 'w'))
reference_sequence_path = updated_reference_path
# copy final file
copyfile(reference_sequence_path, temp_dir_path + args.corrected)
return
def main(args):
# parse args
args = parse_args()
command_line = " ".join(sys.argv[:])
print("Command Line: {cmdLine}\n".format(cmdLine=command_line),
file=sys.stderr)
# get absolute paths to inputs
args.files_dir = resolvePath(args.files_dir)
args.forward_reference = resolvePath(args.forward_ref)
args.backward_reference = resolvePath(args.backward_ref)
args.out = resolvePath(args.out)
args.bwa_reference = resolvePath(args.bwa_reference)
args.in_T_Hmm = resolvePath(args.in_T_Hmm)
args.in_C_Hmm = resolvePath(args.in_C_Hmm)
args.templateHDP = resolvePath(args.templateHDP)
args.complementHDP = resolvePath(args.complementHDP)
args.fofn = resolvePath(args.fofn)
args.target_regions = resolvePath(args.target_regions)
args.ambiguity_positions = resolvePath(args.ambiguity_positions)
start_message = """
# Starting Signal Align
# Aligning files from: {fileDir}
# Aligning to reference: {reference}
# Aligning maximum of {nbFiles} files
# Using model: {model}
# Using banding: True
# Aligning to regions in: {regions}
# Non-default template HMM: {inThmm}
# Non-default complement HMM: {inChmm}
# Template HDP: {tHdp}
# Complement HDP: {cHdp}
""".format(fileDir=args.files_dir,
reference=args.bwa_reference,
nbFiles=args.nb_files,
inThmm=args.in_T_Hmm,
inChmm=args.in_C_Hmm,
model=args.stateMachineType,
regions=args.target_regions,
tHdp=args.templateHDP,
cHdp=args.complementHDP)
print(start_message, file=sys.stdout)
if args.files_dir is None and args.fofn is None:
print("Need to provide directory with .fast5 files of fofn",
file=sys.stderr)
sys.exit(1)
if not os.path.isfile(args.bwa_reference):
print("Did not find valid reference file, looked for it {here}".format(
here=args.bwa_reference),
file=sys.stderr)
sys.exit(1)
# make directory to put temporary files
temp_folder = FolderHandler()
temp_dir_path = temp_folder.open_folder(args.out + "/tempFiles_alignment")
#
if not args.forward_reference or not args.backward_reference:
args.forward_reference, args.backward_reference = processReferenceFasta(
fasta=args.bwa_reference,
motifs=args.motifs,
work_folder=temp_folder,
positions_file=args.ambiguity_positions)
# list of read files
if args.fofn is not None:
fast5s = [x for x in parseFofn(args.fofn) if x.endswith(".fast5")]
else:
fast5s = [
"/".join([args.files_dir, x]) for x in os.listdir(args.files_dir)
if x.endswith(".fast5")
]
nb_files = args.nb_files
if nb_files < len(fast5s):
shuffle(fast5s)
fast5s = fast5s[:nb_files]
# change paths to the source directory
os.chdir(signalAlignSourceDir())
alignment_args = {
"destination": temp_dir_path,
"stateMachineType": args.stateMachineType,
"bwa_reference": args.bwa_reference,
"in_templateHmm": args.in_T_Hmm,
"in_complementHmm": args.in_C_Hmm,
"in_templateHdp": args.templateHDP,
"in_complementHdp": args.complementHDP,
"output_format": args.outFmt,
"threshold": args.threshold,
"diagonal_expansion": args.diag_expansion,
"constraint_trim": args.constraint_trim,
"degenerate": getDegenerateEnum(args.degenerate),
"twoD_chemistry": args.twoD,
"target_regions": args.target_regions,
"embed": args.embed,
"event_table": args.event_table,
"backward_reference": args.backward_reference,
"forward_reference": args.forward_reference,
"alignment_file": None,
"check_for_temp_file_existance": True,
"track_memory_usage": False,
"get_expectations": False,
}
print("[runSignalAlign]:NOTICE: Got {} files to align".format(len(fast5s)),
file=sys.stdout)
# setup workers for multiprocessing
multithread_signal_alignment(alignment_args, fast5s, args.nb_jobs)
print("\n# signalAlign - finished alignments\n", file=sys.stderr)
print("\n# signalAlign - finished alignments\n", file=sys.stdout)
def new_working_folder(self, append):
"""Create new working folder in order to keep track of each new run of analysis"""
self.working_folder = FolderHandler()
self.working_path = self.working_folder.open_folder(
os.path.join(self.args.output_dir,
"tempFiles_trainModels_" + str(append)))
def main(args):
# parse args
args = parse_args()
command_line = " ".join(sys.argv[:])
print("Command Line: {cmdLine}\n".format(cmdLine=command_line), file=sys.stderr)
start_message = """
# Starting Zayante Error-Correction
# Aligning files from: {fileDir}
# Aligning to reference: {reference}
# Aligning maximum of {nbFiles} files
# Using model: {model}
# Using banding: {banding}
# Aligning to regions in: {regions}
# Non-default template HMM: {inThmm}
# Non-default complement HMM: {inChmm}
# Template HDP: {tHdp}
# Complement HDP: {cHdp}
""".format(fileDir=args.files_dir, reference=args.ref, nbFiles=args.nb_files, banding=args.banded,
inThmm=args.in_T_Hmm, inChmm=args.in_C_Hmm, model=args.stateMachineType, regions=args.target_regions,
tHdp=args.templateHDP, cHdp=args.complementHDP)
print(start_message, file=sys.stdout)
if not os.path.isfile(args.ref):
print("Did not find valid reference file", file=sys.stderr)
sys.exit(1)
temp_folder = FolderHandler()
temp_dir_path = temp_folder.open_folder(args.out + "tempFiles_errorCorrection")
reference_sequence = args.ref
STEP = 10
for cycle in range(0, 8):
for it in range(0, STEP):
# make paths for reference files
forward_reference = temp_folder.add_file_path("forward_reference.{cycle}.{iter}.txt".format(cycle=cycle,
iter=it))
backward_reference = temp_folder.add_file_path("backward_reference.{cycle}.{iter}.txt".format(cycle=cycle,
iter=it))
# make N-ed reference sequence for this iteration
deg, reference_sequence_length = make_degenerate_reference(reference_sequence, it,
forward_reference, backward_reference,
step=STEP)
assert deg, "Problem making degenerate reference for cycle {cycle} iteration {iter}" \
"".format(cycle=cycle, iter=it)
# index the reference for bwa
print("signalAlign - indexing reference", file=sys.stderr)
bwa_ref_index = get_bwa_index(args.ref, temp_dir_path)
print("signalAlign - indexing reference, done", file=sys.stderr)
# setup workers for multiprocessing
workers = args.nb_jobs
work_queue = Manager().Queue()
done_queue = Manager().Queue()
jobs = []
# list of alignment files
fast5s = [x for x in os.listdir(args.files_dir) if x.endswith(".fast5")]
# take only some
if args.nb_files < len(fast5s):
shuffle(fast5s)
fast5s = fast5s[:args.nb_files]
for fast5 in fast5s:
alignment_args = {
"forward_reference": forward_reference,
"backward_reference": backward_reference,
"path_to_EC_refs": None,
"destination": temp_dir_path,
"stateMachineType": args.stateMachineType,
"bwa_index": bwa_ref_index,
"in_templateHmm": args.in_T_Hmm,
"in_complementHmm": args.in_C_Hmm,
"in_templateHdp": args.templateHDP,
"in_complementHdp": args.complementHDP,
"banded": args.banded,
"sparse_output": True,
"in_fast5": args.files_dir + fast5,
"threshold": args.threshold,
"diagonal_expansion": args.diag_expansion,
"constraint_trim": args.constraint_trim,
"target_regions": None,
"degenerate": degenerate_enum(args.degenerate),
}
#alignment = SignalAlignment(**alignment_args)
#alignment.run()
work_queue.put(alignment_args)
for w in range(workers):
p = Process(target=aligner, args=(work_queue, done_queue))
p.start()
jobs.append(p)
work_queue.put('STOP')
for p in jobs:
p.join()
done_queue.put('STOP')
print("\n# signalAlign - finished alignments\n", file=sys.stderr)
print("\n# signalAlign - finished alignments\n", file=sys.stdout)
print("\n# Starting Variant Calling\n", file=sys.stdout)
print("\n# Starting Variant Calling\n", file=sys.stderr)
# cull the alignment files
alns, forward_mask = get_alignments_labels_and_mask(temp_dir_path + "*.tsv", args.nb_files)
degenerate_positions = {
'forward': list(range(it, reference_sequence_length, STEP)),
'backward': list(range(it, reference_sequence_length, STEP)) }
variant_call_file = temp_folder.add_file_path("variants.{cycle}.{iter}.calls".format(cycle=cycle, iter=it))
for aln, forward_bool in zip(alns, forward_mask):
call_methyl_args = {
"sequence": None,
"alignment_file": aln,
"forward": forward_bool,
"out_file": variant_call_file,
"positions": degenerate_positions,
"degenerate_type": degenerate_enum(args.degenerate),
}
#c = CallMethylation(**call_methyl_args)
#c.write()
work_queue.put(call_methyl_args)
for w in range(workers):
p = Process(target=run_methyl_caller, args=(work_queue, done_queue))
p.start()
jobs.append(p)
work_queue.put('STOP')
for p in jobs:
p.join()
done_queue.put('STOP')
print("\n# Finished Variant Calling\n", file=sys.stdout)
print("\n# Finished Variant Calling\n", file=sys.stderr)
new_ref = update_reference(variant_call_file, reference_sequence, 0)
ref_path = temp_folder.add_file_path("iteration.{cycle}.{iter}.fa".format(cycle=cycle, iter=it))
write_fasta("iteration.{cycle}.{iter}.fa".format(cycle=cycle, iter=it), new_ref, open(ref_path, 'w'))
reference_sequence = ref_path
# remove old alignments
for f in glob.glob(temp_dir_path + "*.tsv"):
os.remove(f)
STEP -= 1
return
def test_get_sample_kmers(self):
with tempfile.TemporaryDirectory() as tempdir:
# create fast5 dir
test_fast5 = os.path.join(tempdir, "test.fast5")
copyfile(self.fast5_paths[0], test_fast5)
# create fofn
test_out = os.path.join(tempdir, "test.hdp.tsv")
test_args = create_sa_sample_args(
fast5_dirs=[tempdir],
name="some_name",
fw_reference=self.ecoli_reference,
bwa_reference=self.ecoli_reference,
number_of_kmer_assignments=1,
probability_threshold=0,
kmers_from_reference=False)
working_folder = FolderHandler()
working_folder.open_folder(os.path.join(tempdir, "test_dir"))
sample = SignalAlignSample(working_folder=working_folder,
**test_args)
sample.analysis_files = [
self.assignment_file, self.assignment_file
]
hdp_data_handle = CreateHdpTrainingData([sample],
test_out,
template=True,
complement=False,
verbose=False)
kmers = hdp_data_handle.get_sample_kmers(sample)
self.assertEqual(
kmers, {x
for x in all_string_permutations("ATGC", length=6)})
test_args = create_sa_sample_args(
fast5_dirs=[tempdir],
name="some_name",
fw_reference=self.ecoli_reference,
bwa_reference=self.ecoli_reference,
number_of_kmer_assignments=1,
probability_threshold=0,
kmers_from_reference=False,
motifs=[["ATGC", "ETGC"]])
working_folder = FolderHandler()
working_folder.open_folder(os.path.join(tempdir, "test_dir"))
sample = SignalAlignSample(working_folder=working_folder,
**test_args)
sample.analysis_files = [
self.assignment_file, self.assignment_file
]
hdp_data_handle = CreateHdpTrainingData([sample],
test_out,
template=True,
complement=False,
verbose=False)
kmers = hdp_data_handle.get_sample_kmers(sample)
self.assertEqual(
kmers,
get_motif_kmers(["ATGC", "ETGC"], 6, alphabet="ATGC")
| {x
for x in all_string_permutations("ATGC", length=6)})
test_args = create_sa_sample_args(
fast5_dirs=[tempdir],
name="some_name",
fw_reference=self.ecoli_reference,
bwa_reference=self.ecoli_reference,
number_of_kmer_assignments=1,
probability_threshold=0,
kmers_from_reference=True,
motifs=[["ATGC", "ETGC"]])
working_folder = FolderHandler()
working_folder.open_folder(os.path.join(tempdir, "test_dir"))
sample = SignalAlignSample(working_folder=working_folder,
**test_args)
sample.analysis_files = [
self.assignment_file, self.assignment_file
]
hdp_data_handle = CreateHdpTrainingData([sample],
test_out,
template=True,
complement=False,
verbose=False)
kmers = hdp_data_handle.get_sample_kmers(sample)
expected_kmers = set()
for _, _, sequence in read_fasta(self.ecoli_reference):
expected_kmers |= get_sequence_kmers(sequence,
k=6,
rev_comp=True)
self.assertEqual(
kmers,
get_motif_kmers(["ATGC", "ETGC"], 6, alphabet="ATGC")
| expected_kmers)
def main(args):
# parse args
args = parse_args()
command_line = " ".join(sys.argv[:])
print("Command Line: {cmdLine}\n".format(cmdLine=command_line),
file=sys.stderr)
start_message = """
# Starting Empire Error-Correction
# Aligning files from: {fileDir}
# Aligning to reference: {reference}
# Aligning maximum of {nbFiles} files
# Using model: {model}
# Using banding: {banding}
# Aligning to regions in: {regions}
# Non-default template HMM: {inThmm}
# Non-default complement HMM: {inChmm}
# Template HDP: {tHdp}
# Complement HDP: {cHdp}
""".format(fileDir=args.files_dir,
reference=args.ref,
nbFiles=args.nb_files,
banding=args.banded,
inThmm=args.in_T_Hmm,
inChmm=args.in_C_Hmm,
model=args.stateMachineType,
regions=args.target_regions,
tHdp=args.templateHDP,
cHdp=args.complementHDP)
print(start_message, file=sys.stdout)
if not os.path.isfile(args.ref):
print("Did not find valid reference file", file=sys.stderr)
sys.exit(1)
temp_folder = FolderHandler()
temp_dir_path = temp_folder.open_folder(args.out +
"tempFiles_errorCorrection")
reference_sequence = args.ref
for cycle in range(0, args.cycles):
check, reference_sequence_length = write_degenerate_reference_set(
input_fasta=reference_sequence, out_path=temp_dir_path, step=STEP)
assert check, "Problem making degenerate reference sequence set"
# index the reference for bwa
print("signalAlign - indexing reference", file=sys.stderr)
bwa_ref_index = get_bwa_index(reference_sequence, temp_dir_path)
print("signalAlign - indexing reference, done", file=sys.stderr)
# setup workers for multiprocessing
workers = args.nb_jobs
work_queue = Manager().Queue()
done_queue = Manager().Queue()
jobs = []
# list of alignment files
fast5s = [
x for x in os.listdir(args.files_dir) if x.endswith(".fast5")
]
# take only some
if args.nb_files < len(fast5s):
shuffle(fast5s)
fast5s = fast5s[:args.nb_files]
for fast5 in fast5s:
alignment_args = {
"forward_reference": None,
"backward_reference": None,
"path_to_EC_refs": temp_dir_path,
"destination": temp_dir_path,
"stateMachineType": args.stateMachineType,
"bwa_index": bwa_ref_index,
"in_templateHmm": args.in_T_Hmm,
"in_complementHmm": args.in_C_Hmm,
"in_templateHdp": args.templateHDP,
"in_complementHdp": args.complementHDP,
"banded": args.banded,
"sparse_output": True,
"in_fast5": args.files_dir + fast5,
"threshold": args.threshold,
"diagonal_expansion": args.diag_expansion,
"constraint_trim": args.constraint_trim,
"target_regions": None,
"degenerate": degenerate_enum(args.degenerate),
}
#alignment = SignalAlignment(**alignment_args)
#alignment.run()
work_queue.put(alignment_args)
for w in range(workers):
p = Process(target=aligner, args=(work_queue, done_queue))
p.start()
jobs.append(p)
work_queue.put('STOP')
for p in jobs:
p.join()
done_queue.put('STOP')
print("\n# signalAlign - finished alignments\n", file=sys.stderr)
print("\n# signalAlign - finished alignments\n", file=sys.stdout)
# working sequence is a string, that has the reference we're going to update this cycle
working_sequence = get_first_sequence(reference_sequence)
# register is the relative position that is being N-ed:
# ACGTAGACAATA --> NCGTAGNCAATA = register 0
# ACGTAGACAATA --> ANGTAGANAATA = register 1 ...
for register in range(0, STEP):
print("# Starting Variant Calling, register: {}...".format(
register),
file=sys.stdout,
end='\n')
print("# Starting Variant Calling, register: {}...".format(
register),
file=sys.stderr,
end='')
# cull the alignment files for this register
alns, forward_mask = get_alignments_labels_and_mask(
path_to_alignments=temp_dir_path + "*.tsv.{}".format(register),
max=args.nb_files,
suffix=".{}".format(register))
# this is the list of positions that we're going to look at, based on this register
degenerate_positions = {
'forward':
list(range(register, reference_sequence_length, STEP)),
'backward':
list(range(register, reference_sequence_length, STEP))
}
# place to put the marginal probs
variant_call_file = temp_folder.add_file_path(
"variants.{cycle}.{reg}.calls".format(cycle=cycle,
reg=register))
# arguments for multiprocessing
for aln, forward_bool in zip(alns, forward_mask):
call_methyl_args = {
"sequence": None,
"alignment_file": aln,
"forward": forward_bool,
"out_file": variant_call_file,
"positions": degenerate_positions,
"degenerate_type": degenerate_enum(args.degenerate),
}
#c = CallMethylation(**call_methyl_args)
#c.write()
work_queue.put(call_methyl_args)
for w in range(workers):
p = Process(target=run_methyl_caller,
args=(work_queue, done_queue))
p.start()
jobs.append(p)
work_queue.put('STOP')
for p in jobs:
p.join()
done_queue.put('STOP')
# this is where the per-register update happens
working_sequence = update_reference(variant_call_file,
working_sequence,
register,
min_depth=0,
get_sites=False)
# remove alignments for this register
for f in glob.glob(temp_dir_path + "*.tsv.{}".format(register)):
os.remove(f)
print("done", file=sys.stdout, end="\n")
print("done", file=sys.stderr, end="\n")
# add a file for this cycle
ref_path = temp_folder.add_file_path(
"iteration.{cycle}.fa".format(cycle=cycle))
# write it to a file
write_fasta("iteration.{cycle}.fa".format(cycle=cycle),
working_sequence, open(ref_path, 'w'))
# update the path to the reference for the next cycle
reference_sequence = ref_path
return
def main(args):
# parse args
args = parse_args()
command_line = " ".join(sys.argv[:])
print("Command Line: {cmdLine}\n".format(cmdLine=command_line), file=sys.stderr)
# get absolute paths to inputs
args.files_dir = resolvePath(args.files_dir)
args.ref = resolvePath(args.ref)
args.out = resolvePath(args.out)
args.bwt = resolvePath(args.bwt)
args.in_T_Hmm = resolvePath(args.in_T_Hmm)
args.in_C_Hmm = resolvePath(args.in_C_Hmm)
args.templateHDP = resolvePath(args.templateHDP)
args.complementHDP = resolvePath(args.complementHDP)
args.fofn = resolvePath(args.fofn)
args.target_regions = resolvePath(args.target_regions)
args.ambiguity_positions = resolvePath(args.ambiguity_positions)
start_message = """
# Starting Signal Align
# Aligning files from: {fileDir}
# Aligning to reference: {reference}
# Aligning maximum of {nbFiles} files
# Using model: {model}
# Using banding: True
# Aligning to regions in: {regions}
# Non-default template HMM: {inThmm}
# Non-default complement HMM: {inChmm}
# Template HDP: {tHdp}
# Complement HDP: {cHdp}
""".format(fileDir=args.files_dir, reference=args.ref, nbFiles=args.nb_files,
inThmm=args.in_T_Hmm, inChmm=args.in_C_Hmm, model=args.stateMachineType, regions=args.target_regions,
tHdp=args.templateHDP, cHdp=args.complementHDP)
print(start_message, file=sys.stdout)
if args.files_dir is None and args.fofn is None:
print("Need to provide directory with .fast5 files of fofn", file=sys.stderr)
sys.exit(1)
if not os.path.isfile(args.ref):
print("Did not find valid reference file, looked for it {here}".format(here=args.ref), file=sys.stderr)
sys.exit(1)
# make directory to put temporary files
temp_folder = FolderHandler()
temp_dir_path = temp_folder.open_folder(args.out + "/tempFiles_alignment")
#
forward_reference, backward_reference = processReferenceFasta(fasta=args.ref,
motif_key=args.motif_key,
work_folder=temp_folder,
sub_char=args.ambig_char,
positions_file=args.ambiguity_positions)
# index the reference for bwa
if args.bwt is not None:
print("[RunSignalAlign]NOTICE - using provided BWT %s" % args.bwt)
bwa_ref_index = args.bwt
else:
print("signalAlign - indexing reference", file=sys.stderr)
bwa_ref_index = getBwaIndex(args.ref, temp_dir_path)
print("signalAlign - indexing reference, done", file=sys.stderr)
# setup workers for multiprocessing
workers = args.nb_jobs
work_queue = Manager().Queue()
done_queue = Manager().Queue()
jobs = []
# list of read files
if args.fofn is not None:
fast5s = [x for x in parseFofn(args.fofn) if x.endswith(".fast5")]
else:
fast5s = ["/".join([args.files_dir, x]) for x in os.listdir(args.files_dir) if x.endswith(".fast5")]
nb_files = args.nb_files
if nb_files < len(fast5s):
shuffle(fast5s)
fast5s = fast5s[:nb_files]
# change paths to the source directory
os.chdir(signalAlignSourceDir())
print("[runSignalAlign]:NOTICE: Got {} files to align".format(len(fast5s)), file=sys.stdout)
for fast5 in fast5s:
print(fast5)
alignment_args = {
"destination": temp_dir_path,
"stateMachineType": args.stateMachineType,
"bwa_index": bwa_ref_index,
"in_templateHmm": args.in_T_Hmm,
"in_complementHmm": args.in_C_Hmm,
"in_templateHdp": args.templateHDP,
"in_complementHdp": args.complementHDP,
"output_format": args.outFmt,
"in_fast5": fast5,
"threshold": args.threshold,
"diagonal_expansion": args.diag_expansion,
"constraint_trim": args.constraint_trim,
"degenerate": getDegenerateEnum(args.degenerate),
"twoD_chemistry": args.twoD,
"target_regions": args.target_regions,
"embed": args.embed,
"event_table": args.event_table,
"backward_reference": backward_reference,
"forward_reference": forward_reference
}
if args.DEBUG:
alignment = SignalAlignment(**alignment_args)
alignment.run()
else:
work_queue.put(alignment_args)
for w in range(workers):
p = Process(target=aligner, args=(work_queue, done_queue))
p.start()
jobs.append(p)
work_queue.put('STOP')
for p in jobs:
p.join()
done_queue.put('STOP')
print("\n# signalAlign - finished alignments\n", file=sys.stderr)
print("\n# signalAlign - finished alignments\n", file=sys.stdout)
def trainModelTransitions(config):
def process_sample(sample):
options = dict(**DEFAULT_TRAINMODELS_OPTIONS)
options.update(sample)
if options["fast5_dir"] is None and options["fofn"] is None:
raise RuntimeError(
"Need to provide path to .fast5 files or file with filenames (fofn)"
)
reference_map = processReferenceFasta(
fasta=config["reference"],
work_folder=working_folder,
motif_key=options["motif"],
sub_char=options["label"],
positions_file=options["positions_file"])
if options["fast5_dir"] is not None:
if options["fofn"] is not None:
print(
"WARNING Only using files is directory %s ignoring fofn %s"
% (options["files_dir"], options["fofn"]))
sample = Fast5Directory(options["fast5_dir"], reference_map)
else:
sample = FileOfFilenames(options["fofn"], reference_map)
return sample
# make directory to put the files we're using
working_folder = FolderHandler()
working_folder_path = working_folder.open_folder(config["output_dir"] +
"temp_trainModels")
samples = [process_sample(s) for s in config["samples"]]
if config["bwt"] is not None:
print("[trainModels]Using provided BWT")
bwa_ref_index = config["bwt"]
else:
print("signalAlign - indexing reference", file=sys.stderr)
bwa_ref_index = getBwaIndex(config["reference"], working_folder_path)
print("signalAlign - indexing reference, done", file=sys.stderr)
template_model_path = config["in_T_Hmm"]
complement_model_path = config["in_C_Hmm"]
assert os.path.exists(template_model_path) and os.path.exists(complement_model_path), \
"Missing input models %s and %s" % (template_model_path, complement_model_path)
template_model = get_model(config["stateMachineType"], template_model_path)
complement_model = get_model(
config["stateMachineType"],
complement_model_path) if config["twoD"] else None
# get the input HDP, if we're using it
if config["stateMachineType"] == "threeStateHdp":
template_hdp = working_folder.add_file_path(
"%s" % config["templateHdp"].split("/")[-1])
copyfile(config["templateHdp"], template_hdp)
if config["twoD"]:
complement_hdp = working_folder.add_file_path(
"%s" % config["complementHdp"].split("/")[-1])
copyfile(config["complementHdp"], complement_hdp)
else:
complement_hdp = None
else:
template_hdp = None
complement_hdp = None
# make some paths to files to hold the HMMs
template_hmm = working_folder.add_file_path("template_trained.hmm")
complement_hmm = working_folder.add_file_path("complement_trained.hmm")
trained_models = [template_hmm, complement_hmm]
untrained_models = [template_model_path, complement_model_path]
for default_model, trained_model in zip(untrained_models, trained_models):
assert os.path.exists(
default_model), "Didn't find default model {}".format(
default_model)
copyfile(default_model, trained_model)
assert os.path.exists(
trained_model), "Problem copying default model to {}".format(
trained_model)
# start iterating
i = 0
while i < config["iterations"]:
# first cull a set of files to get expectations on
training_files = cull_training_files(
samples=samples,
training_amount=config["training_bases"],
twoD=config["twoD"])
# setup
workers = config["job_count"]
work_queue = Manager().Queue()
done_queue = Manager().Queue()
jobs = []
# get expectations for all the files in the queue
# file_ref_tuple should be (fast5, (plus_ref_seq, minus_ref_seq))
for fast5, ref_map in training_files:
alignment_args = {
"reference_map": ref_map,
"destination": working_folder_path,
"stateMachineType": config["stateMachineType"],
"bwa_index": bwa_ref_index,
"in_templateHmm": template_hmm,
"in_complementHmm": complement_hmm,
"in_templateHdp": template_hdp,
"in_complementHdp": complement_hdp,
"in_fast5": fast5,
"threshold": 0.01,
"diagonal_expansion": config["diagonal_expansion"],
"constraint_trim": config["constraint_trim"],
"target_regions": None,
"degenerate": None,
"twoD_chemistry": config["twoD"],
}
if config["DEBUG"]:
alignment = SignalAlignment(**alignment_args)
alignment.run(get_expectations=True)
else:
work_queue.put(alignment_args)
for w in xrange(workers):
p = Process(target=get_expectations, args=(work_queue, done_queue))
p.start()
jobs.append(p)
work_queue.put('STOP')
for p in jobs:
p.join()
done_queue.put('STOP')
# load then normalize the expectations
template_expectations_files = [
x for x in os.listdir(working_folder_path)
if x.endswith(".template.expectations")
]
complement_expectations_files = [
x for x in os.listdir(working_folder_path)
if x.endswith(".complement.expectations")
]
if len(template_expectations_files) > 0:
add_and_norm_expectations(path=working_folder_path,
files=template_expectations_files,
model=template_model,
hmm_file=template_hmm,
update_transitions=True)
if config["twoD"] and len(complement_expectations_files) > 0:
add_and_norm_expectations(path=working_folder_path,
files=complement_expectations_files,
model=complement_model,
hmm_file=complement_hmm,
update_transitions=True)
# log the running likelihood
if len(template_model.running_likelihoods) > 0 and \
(config["twoD"] and len(complement_model.running_likelihoods)) > 0:
print("{i}| {t_likelihood}\t{c_likelihood}".format(
t_likelihood=template_model.running_likelihoods[-1],
c_likelihood=complement_model.running_likelihoods[-1],
i=i))
if config["TEST"] and (len(template_model.running_likelihoods) >= 2) and \
(config["twoD"] and len(complement_model.running_likelihoods) >= 2):
print("TESTING")
assert (template_model.running_likelihoods[-2] < template_model.running_likelihoods[-1]) and \
(complement_model.running_likelihoods[-2] < complement_model.running_likelihoods[-1]), \
"Testing: Likelihood error, went up"
i += 1
# if we're using HDP, trim the final Hmm (remove assignments)
print("trainModels - finished training routine", file=sys.stdout)
print("trainModels - finished training routine", file=sys.stderr)
def main(args):
# parse args
args = parse_args()
command_line = " ".join(sys.argv[:])
print("Command Line: {cmdLine}\n".format(cmdLine=command_line),
file=sys.stderr)
start_message = """
# Starting BonnyDoon Error-Correction
# Aligning files from: {fileDir}
# Aligning to reference: {reference}
# Aligning maximum of {nbFiles} files
# Using model: {model}
# Using banding: {banding}
# Aligning to regions in: {regions}
# Non-default template HMM: {inThmm}
# Non-default complement HMM: {inChmm}
# Template HDP: {tHdp}
# Complement HDP: {cHdp}
""".format(fileDir=args.files_dir,
reference=args.ref,
nbFiles=args.nb_files,
banding=args.banded,
inThmm=args.in_T_Hmm,
inChmm=args.in_C_Hmm,
model=args.stateMachineType,
regions=args.target_regions,
tHdp=args.templateHDP,
cHdp=args.complementHDP)
print(start_message, file=sys.stdout)
# cull the MinION files
fast5s = cull_fast5_files(args.files_dir, args.nb_files)
# get the (input) reference sequence
if not os.path.isfile(args.ref):
print("Did not find valid reference file", file=sys.stderr)
sys.exit(1)
reference_sequence_path = args.ref
# unpack the reference sequence
reference_sequence_string = get_first_sequence(reference_sequence_path)
# make a working folder in the specified directory
temp_folder = FolderHandler()
temp_dir_path = temp_folder.open_folder(args.out +
"tempFiles_errorCorrection")
# index the reference for bwa this is a string with the path to the index
bwa_ref_index = get_bwa_index(reference_sequence_path, temp_dir_path)
# alignment args are the parameters to the HMM/HDP model, and don't change
alignment_args = {
"path_to_EC_refs": None,
"destination": temp_dir_path,
"stateMachineType": args.stateMachineType,
"bwa_index": bwa_ref_index,
"in_templateHmm": args.in_T_Hmm,
"in_complementHmm": args.in_C_Hmm,
"in_templateHdp": args.templateHDP,
"in_complementHdp": args.complementHDP,
"banded": args.banded,
"sparse_output": True,
"threshold": args.threshold,
"diagonal_expansion": args.diag_expansion,
"constraint_trim": args.constraint_trim,
"target_regions": None,
"degenerate": degenerate_enum(args.degenerate),
}
# get the sites that have proposed edits
proposals = scan_for_proposals(temp_folder, STEP,
reference_sequence_string, fast5s,
alignment_args, args.nb_jobs)
proposals = group_sites_in_window2([x[0] for x in proposals], 6)
return
def __init__(
self,
in_fast5,
destination,
stateMachineType,
in_templateHmm,
in_complementHmm,
in_templateHdp,
in_complementHdp,
threshold,
diagonal_expansion,
constraint_trim,
degenerate,
forward_reference,
backward_reference=None,
# one of these needs to be set
alignment_file=None,
bwa_reference=None,
# reasonable defaults
twoD_chemistry=False,
target_regions=None,
output_format="full",
embed=False,
event_table=False,
check_for_temp_file_existance=True,
track_memory_usage=False,
get_expectations=False,
path_to_bin=''):
self.in_fast5 = in_fast5 # fast5 file to align
self.destination = destination # place where the alignments go, should already exist
self.stateMachineType = stateMachineType # flag for signalMachine
self.bwa_reference = bwa_reference # path to reference sequence to generate guide alignment
self.threshold = threshold # min posterior probability to keep
self.diagonal_expansion = diagonal_expansion # alignment algorithm param
self.constraint_trim = constraint_trim # alignment algorithm param
self.output_format = output_format # smaller output files
self.degenerate = degenerate # set of nucleotides for degenerate characters
self.twoD_chemistry = twoD_chemistry # flag for 2D sequencing runs
self.temp_folder = FolderHandler(
) # object for holding temporary files (non-toil)
self.read_name = self.in_fast5.split(
"/")[-1][:-6] # get the name without the '.fast5'
self.target_regions = target_regions
self.output_formats = {"full": 0, "variantCaller": 1, "assignments": 2}
self.embed = embed # embed the output into the fast5 file
self.event_table = event_table # specify which event table to use to generate alignments
self.backward_reference = backward_reference # fasta path to backward reference if modified bases are used
self.forward_reference = forward_reference # fasta path to forward reference
self.alignment_file = alignment_file # guide aligments will be gotten from here if set
self.check_for_temp_file_existance = check_for_temp_file_existance # don't recreate if files exist
self.track_memory_usage = track_memory_usage # has the 'time' program append mem usage stats to output
self.max_memory_usage_kb = None
self.read_label = None
self.get_expectations = get_expectations # option to gather expectations of transitions and emissions
self.path_to_signalMachine = os.path.join(
path_to_bin, "signalMachine") # path to signalMachine
assert os.path.exists(
self.path_to_signalMachine), "Path to signalMachine does not exist"
assert self.bwa_reference is not None or self.alignment_file is not None, \
"either 'bwa_reference' or 'alignment_file' argument is needed to generate cigar strings"
if (in_templateHmm is not None) and os.path.isfile(in_templateHmm):
self.in_templateHmm = in_templateHmm
else:
self.in_templateHmm = None
if (in_complementHmm is not None) and os.path.isfile(in_complementHmm):
self.in_complementHmm = in_complementHmm
else:
self.in_complementHmm = None
# similarly for HDPs
if (in_templateHdp is not None) and os.path.isfile(in_templateHdp):
self.in_templateHdp = in_templateHdp
else:
self.in_templateHdp = None
if (in_complementHdp is not None) and os.path.isfile(in_complementHdp):
self.in_complementHdp = in_complementHdp
else:
self.in_complementHdp = None
assert os.path.exists(self.destination), \
"Destination path does not exist: {}".format(self.destination)
def main(args):
# parse args
start = timer()
args = parse_args()
if args.command == "run":
if not os.path.exists(args.config):
print("{config} not found".format(config=args.config))
exit(1)
# run training
config_args = create_dot_dict(load_json(args.config))
temp_folder = FolderHandler()
temp_dir_path = temp_folder.open_folder(
os.path.join(os.path.abspath(config_args.output_dir),
"tempFiles_alignment"))
temp_dir_path = resolvePath(temp_dir_path)
print(config_args.output_dir)
print(temp_dir_path)
sa_args = [
merge_dicts([
s, {
"quality_threshold": config_args.filter_reads,
"workers": config_args.job_count
}
]) for s in config_args.samples
]
samples = [
SignalAlignSample(working_folder=temp_folder, **s) for s in sa_args
]
copyfile(args.config,
os.path.join(temp_dir_path, os.path.basename(args.config)))
state_machine_type = "threeState"
if config_args.template_hdp_model_path is not None:
state_machine_type = "threeStateHdp"
alignment_args = create_signalAlignment_args(
destination=temp_dir_path,
stateMachineType=state_machine_type,
in_templateHmm=resolvePath(config_args.template_hmm_model),
in_complementHmm=resolvePath(config_args.complement_hmm_model),
in_templateHdp=resolvePath(config_args.template_hdp_model),
in_complementHdp=resolvePath(config_args.complement_hdp_model),
diagonal_expansion=config_args.diagonal_expansion,
constraint_trim=config_args.constraint_trim,
traceBackDiagonals=config_args.traceBackDiagonals,
twoD_chemistry=config_args.two_d,
get_expectations=False,
path_to_bin=resolvePath(config_args.path_to_bin),
check_for_temp_file_existance=True,
threshold=config_args.signal_alignment_args.threshold,
track_memory_usage=config_args.signal_alignment_args.
track_memory_usage,
embed=config_args.signal_alignment_args.embed,
event_table=config_args.signal_alignment_args.event_table,
output_format=config_args.signal_alignment_args.output_format,
filter_reads=config_args.filter_reads,
delete_tmp=config_args.signal_alignment_args.delete_tmp)
multithread_signal_alignment_samples(samples,
alignment_args,
config_args.job_count,
trim=None,
debug=config_args.debug)
print("\n# signalAlign - finished alignments\n", file=sys.stderr)
print("\n# signalAlign - finished alignments\n", file=sys.stdout)
stop = timer()
else:
command_line = " ".join(sys.argv[:])
print(os.getcwd())
print("Command Line: {cmdLine}\n".format(cmdLine=command_line),
file=sys.stderr)
# get absolute paths to inputs
args.files_dir = resolvePath(args.files_dir)
args.forward_reference = resolvePath(args.forward_ref)
args.backward_reference = resolvePath(args.backward_ref)
args.out = resolvePath(args.out)
args.bwa_reference = resolvePath(args.bwa_reference)
args.in_T_Hmm = resolvePath(args.in_T_Hmm)
args.in_C_Hmm = resolvePath(args.in_C_Hmm)
args.templateHDP = resolvePath(args.templateHDP)
args.complementHDP = resolvePath(args.complementHDP)
args.fofn = resolvePath(args.fofn)
args.target_regions = resolvePath(args.target_regions)
args.ambiguity_positions = resolvePath(args.ambiguity_positions)
args.alignment_file = resolvePath(args.alignment_file)
start_message = """
# Starting Signal Align
# Aligning files from: {fileDir}
# Aligning to reference: {reference}
# Aligning maximum of {nbFiles} files
# Using model: {model}
# Using banding: True
# Aligning to regions in: {regions}
# Non-default template HMM: {inThmm}
# Non-default complement HMM: {inChmm}
# Template HDP: {tHdp}
# Complement HDP: {cHdp}
""".format(fileDir=args.files_dir,
reference=args.bwa_reference,
nbFiles=args.nb_files,
inThmm=args.in_T_Hmm,
inChmm=args.in_C_Hmm,
model=args.stateMachineType,
regions=args.target_regions,
tHdp=args.templateHDP,
cHdp=args.complementHDP)
print(start_message, file=sys.stdout)
if args.files_dir is None and args.fofn is None:
print("Need to provide directory with .fast5 files of fofn",
file=sys.stderr)
sys.exit(1)
if not os.path.isfile(args.bwa_reference):
print("Did not find valid reference file, looked for it {here}".
format(here=args.bwa_reference),
file=sys.stderr)
sys.exit(1)
# make directory to put temporary files
if not os.path.isdir(args.out):
print("Creating output directory: {}".format(args.out),
file=sys.stdout)
os.mkdir(args.out)
temp_folder = FolderHandler()
temp_dir_path = temp_folder.open_folder(
os.path.join(os.path.abspath(args.out), "tempFiles_alignment"))
temp_dir_path = resolvePath(temp_dir_path)
print(args.out)
print(temp_dir_path)
# generate reference sequence if not specified
if not args.forward_reference or not args.backward_reference:
args.forward_reference, args.backward_reference = processReferenceFasta(
fasta=args.bwa_reference,
work_folder=temp_folder,
positions_file=args.ambiguity_positions,
name="")
# list of read files
if args.fofn is not None:
fast5s = [x for x in parseFofn(args.fofn) if x.endswith(".fast5")]
else:
fast5s = [
"/".join([args.files_dir, x])
for x in os.listdir(args.files_dir) if x.endswith(".fast5")
]
nb_files = args.nb_files
if nb_files < len(fast5s):
shuffle(fast5s)
fast5s = fast5s[:nb_files]
# return alignment_args
alignment_args = {
"destination": temp_dir_path,
"stateMachineType": args.stateMachineType,
"bwa_reference": args.bwa_reference,
"in_templateHmm": args.in_T_Hmm,
"in_complementHmm": args.in_C_Hmm,
"in_templateHdp": args.templateHDP,
"in_complementHdp": args.complementHDP,
"output_format": args.outFmt,
"threshold": args.threshold,
"diagonal_expansion": args.diag_expansion,
"constraint_trim": args.constraint_trim,
"degenerate": getDegenerateEnum(args.degenerate),
"twoD_chemistry": args.twoD,
"target_regions": args.target_regions,
"embed": args.embed,
"event_table": args.event_table,
"backward_reference": args.backward_reference,
"forward_reference": args.forward_reference,
"alignment_file": args.alignment_file,
"check_for_temp_file_existance": True,
"track_memory_usage": False,
"get_expectations": False,
"perform_kmer_event_alignment": args.perform_kmer_event_alignment,
"enforce_supported_versions": args.enforce_supported_versions,
"filter_reads": 7 if args.filter_reads else None,
"path_to_bin": args.path_to_bin,
"delete_tmp": args.delete_tmp
}
filter_read_generator = None
if args.filter_reads is not None and args.alignment_file and args.readdb and args.files_dir:
print("[runSignalAlign]:NOTICE: Filtering out low quality reads",
file=sys.stdout)
filter_read_generator = filter_reads_to_string_wrapper(
filter_reads(args.alignment_file,
args.readdb, [args.files_dir],
quality_threshold=7,
recursive=args.recursive))
print("[runSignalAlign]:NOTICE: Got {} files to align".format(
len(fast5s)),
file=sys.stdout)
# setup workers for multiprocessing
multithread_signal_alignment(
alignment_args,
fast5s,
args.nb_jobs,
debug=args.DEBUG,
filter_reads_to_string_wrapper=filter_read_generator)
stop = timer()
print("\n# signalAlign - finished alignments\n", file=sys.stderr)
print("\n# signalAlign - finished alignments\n", file=sys.stdout)
print("[signalAlign] Complete")
print("Running Time = {} seconds".format(stop - start))
def main(args):
# parse args
args = parse_args(args)
command_line = " ".join(sys.argv[:])
print("[singleNucleotideProbabilities] Command Line: {cmdLine}\n".format(
cmdLine=command_line),
file=sys.stderr)
# first: see if we want to validate and return
if args.validation_file is not None:
if os.path.isfile(args.validation_file):
validate_snp_file(args.validation_file,
args.ref,
print_sequences=True,
print_summary=True)
elif os.path.isdir(args.validation_file):
validate_snp_directory(args.validation_file,
args.ref,
print_summary=False,
move_files=False,
make_plots=True,
alignment_file_location=args.alignment_file)
else:
print("[error] got invalid validation location: {}".format(
args.validation_file))
return 0
# get absolute paths to inputs
args.files_dir = resolvePath(args.files_dir)
args.fast5_glob = resolvePath(args.fast5_glob)
args.ref = resolvePath(args.ref)
args.out = resolvePath(args.out)
args.in_T_Hmm = resolvePath(args.in_T_Hmm)
args.in_C_Hmm = resolvePath(args.in_C_Hmm)
args.templateHDP = resolvePath(args.templateHDP)
args.complementHDP = resolvePath(args.complementHDP)
args.target_regions = resolvePath(args.target_regions)
args.alignment_file = resolvePath(args.alignment_file)
# assert integers
args.step_size = int(args.step_size)
args.kmer_size = int(args.kmer_size)
# get input glob
input_glob = args.fast5_glob if args.fast5_glob is not None else os.path.join(
args.files_dir, "*.fast5")
start_message = """
# Single Nucleotide Probabilities
#
# Aligning files matching: {inputGlob}
# Aligning to reference: {reference}
# Aligning maximum of {nbFiles} files
# Using model: {model}
# Using banding: {banding}
# Aligning to regions in: {regions}
# Non-default template HMM: {inThmm}
# Non-default complement HMM: {inChmm}
# Template HDP: {tHdp}
# Complement HDP: {cHdp}
# Kmer size: {kmerSize}
# Step size: {stepSize}
# Alignment File: {alignmentFile}
""".format(inputGlob=input_glob,
reference=args.ref,
banding=args.banded,
nbFiles=args.nb_files,
inThmm=args.in_T_Hmm,
inChmm=args.in_C_Hmm,
model=args.stateMachineType,
regions=args.target_regions,
tHdp=args.templateHDP,
cHdp=args.complementHDP,
kmerSize=args.kmer_size,
stepSize=args.step_size,
alignmentFile=args.alignment_file)
print(start_message, file=sys.stdout)
# prep
if not os.path.isdir(args.out): os.mkdir(args.out)
# get fast5 locations and prune
fast5s = glob.glob(input_glob)
if args.nb_files is not None and args.nb_files < len(fast5s):
print(
"[singleNucleotideProbabilities] pruning {} fast5 files down to configured max {}"
.format(len(fast5s), args.nb_files))
shuffle(fast5s)
fast5s = fast5s[:args.nb_files]
# get the (input) reference sequence
if not os.path.isfile(args.ref):
print(
"[singleNucleotideProbabilities] Did not find valid reference file",
file=sys.stderr)
sys.exit(1)
# make a working folder in the specified directory
temp_folder = FolderHandler()
temp_dir_path = temp_folder.open_folder(
os.path.join(args.out, "tempFiles_errorCorrection"))
# alignment args are the parameters to the HMM/HDP model, and don't change
alignment_args = {
# "path_to_EC_refs": None,
"destination": temp_dir_path,
"stateMachineType": args.stateMachineType,
"bwa_reference": args.ref,
"in_templateHmm": args.in_T_Hmm,
"in_complementHmm": args.in_C_Hmm,
"in_templateHdp": args.templateHDP,
"in_complementHdp": args.complementHDP,
"threshold": args.threshold,
"diagonal_expansion": args.diag_expansion,
"constraint_trim": args.constraint_trim,
"target_regions": None,
"degenerate": getDegenerateEnum("variant"),
"alignment_file": args.alignment_file,
'track_memory_usage': False,
'get_expectations': False
}
# get the sites that have proposed edits
print(
"\n\n[singleNucleotideProbabilities] scanning for proposals with %d fast5s"
% len(fast5s))
output_files = discover_single_nucleotide_probabilities(
args,
temp_folder,
args.kmer_size,
args.ref,
fast5s,
alignment_args,
args.nb_jobs,
args.step_size,
output_directory=args.out)
print("\n[singleNucleotideProbabilities] got {} output files:".format(
len(output_files)))
i = 0
for output_file in output_files:
print("\t{}".format(output_file))
i += 1
if i > 10 and len(output_files) > 10:
print("\t...")
break
#validation
if len(output_files) != 0:
validate_snp_directory(os.path.dirname(output_files[0]),
args.ref,
print_summary=True)
print("\n\n[singleNucleotideProbabilities] fin\n")
return 0