def repredict_structures_for_homol_seqs(
query,
seqs2predict_fasta,
threads=None,
prediction_method=None,
pred_method_params=None,
all_hits_list=None,
seqs2predict_list=None,
use_cm_file=None,
):
"""Run RNA structure prediction based on chosen method and parameters.
"""
default_sim_tr_perc = 90
default_score_tr = 0.0
query_max_len_diff = 0.1
try:
if 'default' == prediction_method:
# do nothing
return None, None, []
elif 'rfam-Rc' == prediction_method:
if use_cm_file is None:
msg = "No CM model. Can't use {}.".format(prediction_method)
ml.warning(msg)
return None, None, [msg]
else:
structures, exec_time = cmmodel_rnafold_c(
seqs2predict_fasta,
use_cm_file,
threads=threads,
params=pred_method_params.get(prediction_method, {}))
return structures, exec_time, []
elif 'rfam-centroid' == prediction_method:
# run cmscan if needed
# run cmfetch
# run cmemit -> homologous seqs
# run centroid_homfold
method_parameters = pred_method_params.get(prediction_method, {})
if use_cm_file is None:
msg = "No CM model. Can't use {}.".format(prediction_method)
ml.warning(msg)
return None, None, [msg]
else:
cep = method_parameters.get('cmemit', '')
if '-u' not in cep:
cep += ' -u'
if '-N' not in cep:
cep += ' -N {}'.format(method_parameters.get('n_seqs', 10))
hf_file = run_cmemit(use_cm_file, params=cep)
structures, exec_time = me_centroid_homfold(
seqs2predict_fasta, hf_file, params=method_parameters)
BA_support.remove_one_file_with_try(hf_file)
return structures, exec_time, []
elif 'rfam-sub' == prediction_method:
if use_cm_file is None:
msg = "No CM model. Can't use {}.".format(prediction_method)
ml.warning(msg)
return None, None, [msg]
else:
ref_structure = extract_ref_from_cm(use_cm_file)
structures, exec_time = rfam_subopt_pred(
seqs2predict_fasta,
ref_structure,
params=pred_method_params.get(prediction_method, None),
threads=threads,
)
return structures, exec_time, []
elif 'rnafold' == prediction_method:
structures, exec_time = rnafold_wrap_for_predict(
seqs2predict_fasta,
params=pred_method_params.get(prediction_method,
{}).get('RNAfold', ''))
return structures, exec_time, []
elif 'fq-sub' == prediction_method:
a, qf = mkstemp(prefix='rba_', suffix='_55', dir=CONFIG.tmpdir)
with os.fdopen(a, 'w') as fd:
fd.write('>query\n{}\n'.format(str(query.seq)))
structures, exec_time = subopt_fold_query(
seqs2predict_fasta,
qf,
params=pred_method_params.get(prediction_method, None),
threads=threads)
BA_support.remove_one_file_with_try(qf)
return structures, exec_time, []
elif 'C-A-sub' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, homologous_seqs, msgs = create_nr_trusted_hits_file_MSA_safe(
all_hits=all_hits_list,
query=query,
sim_threshold_percent=method_parameters.get(
'pred_sim_threshold', default_sim_tr_perc),
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del nr_homo_hits_file
f, homologous_sequence_file = mkstemp(prefix='rba_',
suffix='_64',
dir=CONFIG.tmpdir)
with os.fdopen(f, 'w') as fh:
SeqIO.write(homologous_seqs, fh, 'fasta')
structures, exec_time = subopt_fold_alifold(
seqs2predict_fasta,
homologous_sequence_file,
aligner='clustalo',
params=method_parameters,
threads=threads)
BA_support.remove_one_file_with_try(homologous_sequence_file)
del homologous_sequence_file
del homologous_seqs
return structures, exec_time, msgs
elif 'M-A-sub' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, homologous_seqs, msgs = create_nr_trusted_hits_file_MSA_safe(
all_hits=all_hits_list,
query=query,
sim_threshold_percent=method_parameters.get(
'pred_sim_threshold', default_sim_tr_perc),
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del nr_homo_hits_file
f, homologous_sequence_file = mkstemp(prefix='rba_',
suffix='_65',
dir=CONFIG.tmpdir)
with os.fdopen(f, 'w') as fh:
SeqIO.write(homologous_seqs, fh, 'fasta')
structures, exec_time = subopt_fold_alifold(
seqs2predict_fasta,
homologous_sequence_file,
aligner='muscle',
params=method_parameters,
threads=threads,
)
BA_support.remove_one_file_with_try(homologous_sequence_file)
del homologous_sequence_file
del homologous_seqs
return structures, exec_time, msgs
elif 'C-A-r-Rc' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, _, msgs = create_nr_trusted_hits_file_MSA_safe(
all_hits=all_hits_list,
query=query,
sim_threshold_percent=method_parameters.get(
'pred_sim_threshold', default_sim_tr_perc),
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
structures, exec_time = alifold_refold_prediction(
nr_homo_hits_file,
seqs2predict_fasta,
refold='refold_rnafoldc',
threads=threads,
params=method_parameters,
msa_alg='clustalo')
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del nr_homo_hits_file
return structures, exec_time, msgs
elif 'M-A-r-Rc' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, _, msgs = create_nr_trusted_hits_file_MSA_safe(
all_hits=all_hits_list,
query=query,
sim_threshold_percent=method_parameters.get(
'pred_sim_threshold', default_sim_tr_perc),
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
structures, exec_time = alifold_refold_prediction(
nr_homo_hits_file,
seqs2predict_fasta,
refold='refold_rnafoldc',
threads=threads,
params=method_parameters,
msa_alg='muscle')
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del nr_homo_hits_file
return structures, exec_time, msgs
elif 'C-A-U-r-Rc' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, _, msgs = create_nr_trusted_hits_file_MSA_safe(
all_hits=all_hits_list,
query=query,
sim_threshold_percent=method_parameters.get(
'pred_sim_threshold', default_sim_tr_perc),
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
structures, exec_time = alifold_refold_prediction(
nr_homo_hits_file,
seqs2predict_fasta,
refold='conserved_ss_rnafoldc',
threads=threads,
params=method_parameters,
msa_alg='clustalo')
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del nr_homo_hits_file
return structures, exec_time, msgs
elif 'M-A-U-r-Rc' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, _, msgs = create_nr_trusted_hits_file_MSA_safe(
all_hits=all_hits_list,
query=query,
sim_threshold_percent=method_parameters.get(
'pred_sim_threshold', default_sim_tr_perc),
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
structures, exec_time = alifold_refold_prediction(
nr_homo_hits_file,
seqs2predict_fasta,
refold='conserved_ss_rnafoldc',
threads=threads,
params=method_parameters,
msa_alg='muscle')
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del nr_homo_hits_file
return structures, exec_time, msgs
elif 'centroid' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, _, msgs = create_nr_homolog_hits_file_MSA_unsafe(
all_hits=all_hits_list,
query=query,
sim_threshold_percent=method_parameters.get(
'pred_sim_threshold', default_sim_tr_perc),
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
raw_structures, exec_time = me_centroid_homfold(
seqs2predict_fasta,
nr_homo_hits_file,
params=method_parameters)
# check noncanonical
if prediction_method in pred_method_params and pred_method_params[
prediction_method]:
allow_nc = pred_method_params[prediction_method].get(
'allow_noncanonical', False)
allow_lp = pred_method_params[prediction_method].get(
'allow_lonely_pairs', False)
else:
allow_nc = False
allow_lp = False
if not allow_nc:
for seq in raw_structures:
repstr = find_nc_and_remove(
str(seq.seq), structure=seq.letter_annotations['ss0'])
seq.letter_annotations['ss0'] = repstr
# check lonely basepairs
if not allow_lp:
for seq in raw_structures:
repstr = check_lonely_bp(seq.letter_annotations['ss0'])
seq.letter_annotations['ss0'] = repstr
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del nr_homo_hits_file
return raw_structures, exec_time, msgs
elif 'centroid-fast' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
if query.annotations['ambiguous']:
raise exceptions.AmbiguousQuerySequenceException
raw_structures, exec_time = centroid_homfold_fast(
all_seqs=all_hits_list,
query=query,
all_seqs_fasta=seqs2predict_fasta,
n=method_parameters.get('max_seqs_in_prediction', 10),
centroid_homfold_params=method_parameters,
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff))
# check noncanonical
if prediction_method in pred_method_params and pred_method_params[
prediction_method]:
allow_nc = pred_method_params[prediction_method].get(
'allow_noncanonical', False)
allow_lp = pred_method_params[prediction_method].get(
'allow_lonely_pairs', False)
else:
allow_nc = False
allow_lp = False
if not allow_nc:
for seq in raw_structures:
repstr = find_nc_and_remove(
str(seq.seq), structure=seq.letter_annotations['ss0'])
seq.letter_annotations['ss0'] = repstr
# check lonely basepairs
if not allow_lp:
for seq in raw_structures:
repstr = check_lonely_bp(seq.letter_annotations['ss0'])
seq.letter_annotations['ss0'] = repstr
return raw_structures, exec_time, []
elif 'TurboFold' == prediction_method:
# set arbitrary sim_threshold_percent to 100, because we want to remove only identical sequences from prediction
# with TurboFold. The structure of redundant sequences will be set according to the one in prediction
all_hits_filtered = BA_support.filter_ambiguous_seqs_from_list(
all_hits_list)
seqs2predict_filtered = BA_support.filter_ambiguous_seqs_from_list(
seqs2predict_list)
if len(seqs2predict_list) != len(seqs2predict_filtered):
ml.warning(
'Some sequences contain ambiguous bases - they will not be predicted.'
)
if query.annotations['ambiguous']:
raise exceptions.AmbiguousQuerySequenceException()
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, _, msgs = create_nr_homolog_hits_file_MSA_unsafe(
all_hits=all_hits_filtered,
query=query,
sim_threshold_percent=100,
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
with open(nr_homo_hits_file, 'r') as nrf:
nr_homo_hits = [
seq for seq in SeqIO.parse(nrf, format='fasta')
]
nh = sha1()
nh.update(str(sorted(method_parameters.items())).encode())
nh_str = nh.hexdigest()
structures_t, exec_time = turbofold_with_homologous(
all_sequences=seqs2predict_filtered,
nr_homologous=nr_homo_hits,
params=method_parameters.get('TurboFold', {}),
n=method_parameters.get('max_seqs_in_prediction', 3),
cpu=threads,
pkey=prediction_method,
sha1val=nh_str,
)
structures = BA_support.rebuild_structures_output_from_pred(
seqs2predict_list, structures_t)
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del structures_t
del nr_homo_hits
del nr_homo_hits_file
return structures, exec_time, msgs
elif 'Turbo-fast' == prediction_method:
if query.annotations['ambiguous']:
raise exceptions.AmbiguousQuerySequenceException()
nh = sha1()
nh.update(
str(
sorted(
pred_method_params.get(prediction_method,
{}).items())).encode())
nh_str = nh.hexdigest()
structures_t, exec_time = turbofold_fast(
all_seqs=all_hits_list,
seqs2predict=seqs2predict_list,
query=query,
cpu=threads,
n=pred_method_params.get(prediction_method,
{}).get('max_seqs_in_prediction', 3),
turbofold_params=pred_method_params.get(prediction_method,
{}).get(
'TurboFold', {}),
len_diff=pred_method_params.get(prediction_method, {}).get(
'query_max_len_diff', query_max_len_diff),
pkey=prediction_method,
sha1val=nh_str,
)
structures = BA_support.rebuild_structures_output_from_pred(
seqs2predict_list, structures_t)
del structures_t
return structures, exec_time, []
except exceptions.NoHomologousSequenceException:
msg = nonhomseqwarn(prediction_method)
return None, None, [msg]
except exceptions.AmbiguousQuerySequenceException:
msgfail = "Query sequence contains ambiguous characters. Can't use {}.".format(
prediction_method)
ml.warning(msgfail)
return None, None, [msgfail]
except exceptions.SubprocessException as e:
msg = "{} can't be used. Error message follows: {} \n{}".format(
prediction_method, str(e), e.errors)
ml.error(msg)
return None, None, [str(e)]
except Exception as e:
ml.error("{} can't be used. Error message follows: \n{}.".format(
prediction_method, str(e)))
return None, None, [str(e)]
assert False, "Should not reach here (bad prediction method name)."
def turbofold_ext_nr_fast(all_seqs,
nrset,
cpu,
n,
turbofold_params,
pkey='Turbo-fast',
sha1val='',
timeout=None):
# ambiguos sequence cannot be predicted with turbofold
# do not predict sequence twice
ml.debug(fname())
msg_short_list = 'Turbo-fast: Number of sequences is less then required.'
if not len(nrset) == len(
BA_support.filter_ambiguous_seqs_from_list(nrset)) == len(
BA_support.non_redundant_seqs(nrset)):
msgfail = 'Wrong nr set specification.'
if len(nrset) != len(
BA_support.filter_ambiguous_seqs_from_list(nrset)):
msgfail += 'nr set contains seq(s) with ambiguous character.'
if len(nrset) != len(BA_support.non_redundant_seqs(nrset)):
msgfail += 'nr set contain non redundant sequence(s).'
ml.error(msgfail)
raise AssertionError(msgfail)
for seq in chain(all_seqs, nrset):
if 'msgs' not in seq.annotations:
seq.annotations['msgs'] = []
list2predict = []
for seq in all_seqs:
if seq.annotations.get('ambiguous', False):
ml.warning('Skipping prediction for {} (ambiguous base)'.format(
seq.id))
continue
if pkey in seq.letter_annotations and seq.annotations.get(
'sha1', {}).get(pkey) == sha1val:
# nothing to do, the structure already computed
continue
seq_set = _prepare_set_n(seq, nrset, n)
if len(seq_set) < 2:
msgfail = "Turbo-fast can't be used with less then 2 sequences - {}".format(
seq.id)
ml.warning(msgfail)
if ml.getEffectiveLevel() > 30:
print(msgfail)
continue
if len(seq_set) < n:
msg_short_list_custom = msg_short_list + ' n={} ({})'.format(
len(seq_set), n)
ml.info(msg_short_list_custom + ' ' + seq.id)
seq.annotations['msgs'].append(msg_short_list_custom)
list2predict.append((seq_set, turbofold_params, seq.id))
if cpu == 1:
pred_list = []
for oneseqset, tpar, _ in list2predict:
pred_list.append(run_turbofold(oneseqset, tpar, timeout=timeout))
else:
pool = multiprocessing.Pool(processes=cpu)
pred_list = pool.map(_rt_wrapper, list2predict)
pool.close()
# rebuild predicted TurboFold structures
# - take care that prediction might be empty if TurboFold fails
out_list = []
for out, l_in in zip(pred_list, list2predict):
if isinstance(out, exceptions.SubprocessException):
seq = next(s for s in all_seqs if s.id == l_in[2])
seq.annotations['msgs'].append(str(out))
# do not propagate call output
# seq.annotations['msgs'].append(out.errors)
continue
elif isinstance(out, Exception):
seq = next(s for s in all_seqs if s.id == l_in[2])
seq.annotations['msgs'].append(str(out))
continue
sel = [o for o in out if o.id == l_in[2]]
if len(sel) == 1:
out_list.append(sel[0])
return out_list
def create_nr_trusted_hits_file_MSA_safe(
sim_threshold_percent=None,
all_hits=None,
query=None,
cmscore_tr=-2.03,
cm_threshold_percent=None,
check_unambiguous=False,
len_diff=0.1,
):
"""
create non redundant trusted hits file
multiple at minimum (2) sequences are needed for profile alignment for some aligners
so this function always return two or more sequences or raises exception
:param sim_threshold_percent: seq similarity threshold for homology exclusion
:param all_hits: list of hits
:param query: blast query
:param cmscore_tr: threshold for homology inclusion in bits
:param cm_threshold_percent: threshold for homology inclusion in % of query bits
:param check_unambiguous: bool whether to check unambiguous seqs
:param len_diff: threshold for exclusion of hits lq=len(query) lq - diff*lq < len(seq) < lq + diff*lq
:return:
"""
ml.debug(fname())
# i need to leave query, even if with ambiguous basepairs in
# because it is used as an reference during distance computation and subsequence selection,
# however i don't need to have all homologous seqs there
if check_unambiguous:
all_hits = BA_support.filter_ambiguous_seqs_from_list(all_hits)
dist_table, homologous_seqs, msgs = _trusted_hits_selection_wrapper(
all_hits, query, cmscore_tr, cm_threshold_percent, len_diff_=len_diff)
if dist_table.size == 0:
raise exceptions.NoHomologousSequenceException
to_include = rna_blast_analyze.BR_core.predict_structures.select_sequences_from_similarity_rec(
dist_table, sim_threshold_percent=sim_threshold_percent)
nr_homolog_hits = [homologous_seqs[i] for i in to_include]
# final checking of nr homologs
# if sequence is filtered here, it is ambiguous basepair in query
# removing it is fine if multiple homologous sequences are present
# the problem will arise when only 1 homologous sequence will remain
# if we added sequence in previous step, raise exception, else behave like in prev step
# what if trusted hit is only one?
msg = (
'STATUS: Only one sequence remained under defined "pred_sim_threshold" parameter.\n'
' Mitigation: Adding the most dissimilar homologous sequence to the non redundant sequences list.'
)
if len(nr_homolog_hits) < 2 and not check_unambiguous:
msgs.append(msg)
ml.info(msg)
if ml.level > 20:
print(msg)
dis_hom_index = dist_table[:, 0].argmin()
nr_homolog_hits.append(
SeqRecord(homologous_seqs[dis_hom_index].seq, id='dummy_seq_01'))
del dis_hom_index
elif len(nr_homolog_hits) < 2 and check_unambiguous:
if len(BA_support.filter_ambiguous_seqs_from_list(
nr_homolog_hits)) == 0:
# this mean query contain ambiguous bases
raise exceptions.NoHomologousSequenceException
else:
msgs.append(msg)
ml.info(msg)
if ml.level > 20:
print(msg)
dis_hom_index = dist_table[:, 0].argmin()
nr_homolog_hits.append(
SeqRecord(homologous_seqs[dis_hom_index].seq,
id='dummy_seq_01'))
del dis_hom_index
homologous_seqs = BA_support.filter_ambiguous_seqs_from_list(
homologous_seqs)
elif len(nr_homolog_hits) >= 2 and not check_unambiguous:
pass
elif len(nr_homolog_hits) > 2 and check_unambiguous:
nr_homolog_hits = BA_support.filter_ambiguous_seqs_from_list(
nr_homolog_hits)
homologous_seqs = BA_support.filter_ambiguous_seqs_from_list(
homologous_seqs)
elif len(nr_homolog_hits) == 2 and check_unambiguous:
homologous_seqs = BA_support.filter_ambiguous_seqs_from_list(
homologous_seqs)
if len(BA_support.filter_ambiguous_seqs_from_list(
nr_homolog_hits)) == 1:
# this mean that query contains ambiguous base
raise exceptions.NoHomologousSequenceException
else:
raise Exception()
fd_h, nr_homo_hits_file = mkstemp(prefix='rba_',
suffix='_58',
dir=CONFIG.tmpdir)
with os.fdopen(fd_h, 'w') as f:
SeqIO.write(nr_homolog_hits, f, 'fasta')
return nr_homo_hits_file, homologous_seqs, msgs