def test_create_repeat_list_from_repeats():
test_repeats = [repeat.Repeat(msa=i) for i in TEST_REPEATS]
test_repeat_list = rl.RepeatList(repeats=test_repeats)
assert len(test_repeat_list.repeats) == 4
for i, j in zip(TEST_REPEATS, test_repeat_list.repeats):
assert i == j.msa
def test_serialize_repeat_list_tsv():
test_repeats = [repeat.Repeat(msa=i) for i in TEST_REPEATS[:2]]
test_seq = sequence.Sequence(TEST_SEQUENCE)
for i in test_repeats:
test_seq.repeat_in_sequence(i)
test_repeat_list = rl.RepeatList(repeats=test_repeats)
tsv = test_repeat_list.write("tsv", return_string=True)
assert type(tsv) == str
def test_serialize_repeat_list_tsv():
test_repeats = [repeat.Repeat(msa = i) for i in TEST_REPEATS]
test_seq = sequence.Sequence(TEST_SEQUENCE)
for i in test_repeats:
test_seq.repeat_in_sequence(i)
test_repeat_list = rl.RepeatList(repeats = test_repeats)
tsv = rl_io.serialize_repeat_list_tsv(test_repeat_list)
assert type(tsv) == str
def test_filter_pvalue():
#test_repeats = [repeat.Repeat(msa = i, scoreslist = ["phylo_gap01"], calc_score = True, calc_pvalue = True) for i in TEST_REPEATS]
test_repeats = [repeat.Repeat(msa=i) for i in TEST_REPEATS]
for i, j in zip(test_repeats, TEST_SCORE_VALUE_LIST):
i.d_pvalue = {}
i.d_pvalue[TEST_SCORE] = j
test_repeat_list = rl.RepeatList(repeats=test_repeats)
test_repeat_list_filtered = test_repeat_list.filter(
"pvalue", TEST_SCORE, 0.1)
assert len(test_repeat_list_filtered.repeats) == 1
def test_repeat_list_pickle():
test_repeats = [repeat.Repeat(msa=i) for i in TEST_REPEATS]
test_repeat_list = rl.RepeatList(repeats=test_repeats)
test_pickle = os.path.join(path(), "test.pickle")
test_repeat_list.write('pickle', test_pickle)
test_repeat_list_new = repeat.Repeat.create(test_pickle, 'pickle')
assert len(test_repeat_list.repeats) == len(test_repeat_list_new.repeats)
assert test_repeat_list.repeats[0].msa == test_repeat_list_new.repeats[
0].msa
if os.path.exists(test_pickle):
os.remove(test_pickle)
def test_filter_cluster_based():
test_repeats = [repeat.Repeat(msa=i) for i in TEST_REPEATS]
for i, j in zip(test_repeats, TEST_SCORE_VALUE_LIST):
i.d_pvalue = {}
i.d_pvalue[TEST_SCORE] = j
for i, j in zip(test_repeats, TEST_BEGIN_LIST):
i.begin = j
test_repeat_list = rl.RepeatList(repeats=test_repeats)
test_repeat_list.filter("pvalue", TEST_SCORE, 0.1)
test_repeat_list_filtered = test_repeat_list.filter(
"none_overlapping", ("common_ancestry", None),
[("pvalue", TEST_SCORE), ("divergence", TEST_SCORE)])
assert len(test_repeat_list_filtered.repeats) == 3
for i in test_repeats[:3]:
assert i in test_repeat_list_filtered.repeats
def test_cluster():
test_repeats = [repeat.Repeat(msa=i) for i in TEST_REPEATS]
for i, j in zip(test_repeats, TEST_BEGIN_LIST):
i.begin = j
test_repeat_list = rl.RepeatList(repeats=test_repeats)
test_repeat_list.cluster("common_ancestry")
# Check whether both lists include exactly the same elements.
for i in [{0}, {1, 3}, {2}]:
assert i in test_repeat_list.d_cluster["common_ancestry"]
assert len(test_repeat_list.d_cluster["common_ancestry"]) == 3
test_repeat_list.cluster("shared_char")
# Check whether both lists include exactly the same elements.
for i in [{0}, {1, 2, 3}]:
assert i in test_repeat_list.d_cluster["shared_char"]
assert len(test_repeat_list.d_cluster["shared_char"]) == 2
def workflow(sequences_file,
hmm_annotation_file,
hmm_dir,
result_file,
result_file_serialized,
format,
max_time,
time_interval=3600,
next_time=3600,
**kwargs):
''' Annotate sequences with TRs from multiple sources, test and refine annotations.
Save the annotations in a pickle.
Args:
sequences_file (str): Path to the pickle file containing a list of ``Sequence``
instances.
hmm_dir (str): Path to directory where all HMMs are stored as .pickles
result_file (str): Path to the result file.
max_time (str): Max run time in seconds
Raises:
Exception: If the pickle ``sequences_file`` cannot be loaded
Exception: if the hmm_dir does not exist
'''
start = datetime.datetime.now()
max_time, time_interval, next_time = int(max_time), int(
time_interval), int(next_time)
try:
l_sequence = Fasta(sequences_file)
except:
raise Exception(
"Cannot load putative pickle file sequences_file: {}".format(
sequences_file))
if not os.path.isdir(hmm_dir):
try:
os.makedirs(hmm_dir)
except:
raise Exception(
"hmm_dir does not exists and could not be created: {}".format(
hmm_dir))
try:
with open(hmm_annotation_file, 'rb') as fh:
dHMM_annotation = pickle.load(fh)
except:
raise Exception(
"Cannot load hmm_annotation_file: {}".format(hmm_annotation_file))
basic_filter = CONFIG['filter']['basic']['dict']
basic_filter_tag = CONFIG['filter']['basic']['tag']
# Load previous results
try:
if not os.path.isdir(os.path.dirname(result_file)):
os.makedirs(os.path.dirname(result_file))
except:
raise Exception(
"Could not create path to result_file directory: {}".format(
os.path.dirname(result_file)))
try:
with open(result_file, 'rb') as fh:
dResults = pickle.load(fh)
except:
LOG.debug(
"Could not load previous results file - perhaps non existant: {}".
format(result_file))
dResults = {}
dHMM = {}
for iS_pyfaidx in l_sequence:
# If sequence is already included in results: continue.
if iS_pyfaidx.name in dResults:
continue
elapsed_time = (datetime.datetime.now() - start).seconds
if elapsed_time > max_time or elapsed_time > next_time:
with open(result_file, 'wb') as fh:
pickle.dump(dResults, fh)
next_time = next_time + time_interval
iS = sequence.Sequence(seq=str(iS_pyfaidx), name=iS_pyfaidx.name)
LOG.debug("Work on sequence {}".format(iS))
# 1. annotate_de_novo()
denovo_repeat_list = iS.detect(denovo=True,
repeat={"calc_pvalue": True})
LOG.debug(denovo_repeat_list.repeats)
for iTR in denovo_repeat_list.repeats:
iTR.model = None
# 2. annotate_TRs_from_hmmer()
if iS.name in dHMM_annotation:
lHMM = dHMM_annotation[iS.name]
infoNRuns = len(lHMM)
LOG.debug(
"{} Viterbi runs need to be performed.".format(infoNRuns))
lHMM = set(lHMM)
infoNHMM = len(lHMM)
LOG.debug(
"These derive from {} independent HMMs.".format(infoNHMM))
# Load all HMM pickles needed for the particular sequence.
for hmm_ID in lHMM:
if hmm_ID not in dHMM:
dHMM[hmm_ID] = hmm.HMM.create(file_format="pickle",
file=os.path.join(
hmm_dir,
hmm_ID + ".pickle"))
pfam_repeat_list = iS.detect([dHMM[hmm_ID] for hmm_ID in lHMM],
repeat={"calc_pvalue": True})
for iTR, hmm_ID in zip(pfam_repeat_list.repeats, lHMM):
iTR.model = hmm_ID
iTR.TRD = "PFAM"
else:
pfam_repeat_list = None
# 3. merge_and_basic_filter()
all_repeat_list = denovo_repeat_list + pfam_repeat_list
iS.set_repeatlist(all_repeat_list, REPEAT_LIST_TAG)
iS.set_repeatlist(denovo_repeat_list, DE_NOVO_ALL_TAG)
iS.set_repeatlist(pfam_repeat_list, PFAM_ALL_TAG)
rl_tmp = iS.get_repeatlist(REPEAT_LIST_TAG)
if iS.get_repeatlist(REPEAT_LIST_TAG):
for iB in basic_filter.values():
rl_tmp = rl_tmp.filter(**iB)
else:
rl_tmp = iS.get_repeatlist(REPEAT_LIST_TAG)
iS.set_repeatlist(rl_tmp, basic_filter_tag)
iS.set_repeatlist(rl_tmp.intersection(iS.get_repeatlist(PFAM_ALL_TAG)),
PFAM_TAG)
iS.set_repeatlist(
rl_tmp.intersection(iS.get_repeatlist(DE_NOVO_ALL_TAG)),
DE_NOVO_TAG)
# 4. calculate_overlap()
# Perform common ancestry overlap filter and keep PFAMs
criterion_pfam_fixed = {
"func_name": "none_overlapping_fixed_repeats",
"rl_fixed": iS.get_repeatlist(PFAM_TAG),
"overlap_type": "common_ancestry"
}
iS.d_repeatlist[DE_NOVO_TAG] = iS.get_repeatlist(DE_NOVO_TAG).filter(
**criterion_pfam_fixed)
# Choose only the most convincing de novo TRs
criterion_filter_order = {
"func_name":
"none_overlapping",
"overlap": ("common_ancestry", None),
"l_criterion": [("pvalue", "phylo_gap01"),
("divergence", "phylo_gap01")]
}
iS.d_repeatlist[DE_NOVO_TAG] = iS.get_repeatlist(DE_NOVO_TAG).filter(
**criterion_filter_order)
# 5. refine_denovo()
denovo_final = []
denovo_refined = [None] * len(
iS.get_repeatlist(DE_NOVO_ALL_TAG).repeats)
for i, iTR in enumerate(iS.get_repeatlist(DE_NOVO_ALL_TAG).repeats):
if not iTR in iS.get_repeatlist(DE_NOVO_TAG).repeats:
continue
# Create HMM from TR
denovo_hmm = hmm.HMM.create(file_format='repeat', repeat=iTR)
# Run HMM on sequence
denovo_refined_rl = iS.detect(lHMM=[denovo_hmm])
append_refined = False
if denovo_refined_rl and denovo_refined_rl.repeats:
iTR_refined = denovo_refined_rl.repeats[0]
iTR_refined.TRD = iTR.TRD
iTR_refined.model = "cpHMM"
denovo_refined[i] = iTR_refined
# Check whether new and old TR overlap. Check whether new TR is
# significant. If not both, put unrefined TR into final.
if repeat_list.two_repeats_overlap("shared_char", iTR,
iTR_refined):
rl_tmp = repeat_list.RepeatList([iTR_refined])
LOG.debug(iTR_refined.msa)
for iB in basic_filter.values():
rl_tmp = rl_tmp.filter(**iB)
if rl_tmp.repeats:
append_refined = True
else:
denovo_refined[i] = False
if append_refined:
denovo_final.append(iTR_refined)
else:
denovo_final.append(iTR)
iS.set_repeatlist(repeat_list.RepeatList(denovo_refined),
DE_NOVO_REFINED_TAG)
iS.set_repeatlist(repeat_list.RepeatList(denovo_final),
DE_NOVO_FINAL_TAG)
iS.set_repeatlist(
iS.get_repeatlist(DE_NOVO_FINAL_TAG) + iS.get_repeatlist(PFAM_TAG),
FINAL_TAG)
dResults[iS.name] = iS
# 6.a Save results as pickle
with open(result_file, 'wb') as fh:
pickle.dump(dResults, fh)
# 6.b Save serialized results
with open(result_file_serialized, 'w') as fh_o:
if format == 'tsv':
header = [
"ID", "MSA", "begin", "pvalue", "l_effective", "n",
"n_effective", "TRD", "model"
]
fh_o.write("\t".join(header))
for iS in dResults.values():
for iTR in iS.get_repeatlist(FINAL_TAG).repeats:
if format == 'tsv':
try:
data = [
str(i) for i in [
iS.name, " ".join(iTR.msa), iTR.begin,
iTR.pvalue("phylo_gap01"), iTR.l_effective,
iTR.n, iTR.n_effective, iTR.TRD, iTR.model
]
]
except:
print(iTR)
raise Exception(
"(Could not save data for the above TR.)")
fh_o.write("\n" + "\t".join(data))
print("DONE")
def detect(self, lHMM=None, denovo=None, **kwargs):
""" Detects tandem repeats on ``self.seq`` from 2 possible sources.
A list of ``Repeat`` instances is created for tandem repeat detections
on the sequence from two possible sources:
* Sequence profile hidden Markov models ``HMM``
* de novo detection algorithms.
Args:
hmm (HMM): A list of ``HMM`` instances.
denovo (bool): boolean
*kwargs: Parameters fed to denovo TR prediction and/or Repeat
instantiation. E.g. ``repeat = {"calc_score": True}``
Returns:
A ``RepeatList`` instance
"""
if lHMM:
if not isinstance(lHMM, list):
raise Exception('The lHMM value is not a list.')
for iHMM in lHMM:
if not isinstance(iHMM, hmm.HMM):
raise Exception('At least one list element in the lHMM'
'value is not a valid instance of the HMM'
'class.')
repeats = []
for iHMM in lHMM:
# Detect TRs on self.seq with hmm using the Viterbi algorithm.
most_likely_path = iHMM.viterbi(self.seq)
LOG.debug("most_likely_path: {}".format(most_likely_path))
if not most_likely_path:
continue
unaligned_msa = hmm_viterbi.hmm_path_to_non_aligned_tandem_repeat_units(
self.seq, most_likely_path, iHMM.l_effective)
if len(unaligned_msa) > 1:
# Align the msa
aligned_msa = repeat_align.realign_repeat(unaligned_msa)
if len(aligned_msa) > 1:
# Create a Repeat() class with the new msa
if 'repeat' in kwargs:
repeats.append(
repeat.Repeat(aligned_msa, **kwargs['repeat']))
else:
repeats.append(repeat.Repeat(aligned_msa))
# Set begin coordinate for all repeats
for i_repeat in repeats:
self.repeat_in_sequence(i_repeat)
return repeat_list.RepeatList(repeats)
elif lHMM == []:
LOG.debug("lHMM == []")
return None
elif denovo:
if 'detection' in kwargs:
predicted_repeats = repeat_detection_run.run_detector(
[self], **kwargs['detection'])[0]
else:
predicted_repeats = repeat_detection_run.run_detector([self
])[0]
LOG.debug("predicted_repeats: {}".format(predicted_repeats))
repeats = []
for jTRD, jlTR in predicted_repeats.items():
for iTR in jlTR:
if 'repeat' in kwargs:
iTR = repeat.Repeat(iTR.msa,
begin=iTR.begin,
**kwargs['repeat'])
else:
iTR = repeat.Repeat(iTR.msa, begin=iTR.begin)
# Consider only tandem repeats that have a repeat unit
# predicted to be at least one character long.
if iTR.l_effective > 0:
# Save l, n, MSA, TRD, scores, sequence_type, position
# in sequence of given type
iTR.TRD = jTRD
# Sanity check repeat and set begin coordinate for
# all repeats
if not self.repeat_in_sequence(iTR):
LOG.debug("The tandem repeat is not part of" \
"the sequence. Detector: %s", iTR.TRD)
continue
repeats.append(iTR)
return repeat_list.RepeatList(repeats)
else:
raise Exception("Either require denovo detection, or provide an",
"HMM")