def me_centroid_homfold(fasta2predict, fasta_homologous_seqs, params=None):
"""
run centroid_homefold several times and vary -g parameter, to predict the best possible structure
:param fasta2predict:
:param fasta_homologous_seqs:
:param params:
:return:
"""
# first run centroid homefold for several stages of g (-1)
# find the most stable structure value of g
# structure of output
if params is None:
params = dict()
ch_params = ''
if params and ('centroid_homfold' in params) and params['centroid_homfold']:
ch_params += params['centroid_homfold']
if '-g ' in ch_params and '-g -1' not in ch_params or '-t ' in params:
print("We only allow to run centroid homfold in automatic mode where the structure is predicted with multiple"
" weights and then best scoring structure is selected, threshold's are also forbidden as it implies -g.")
raise AttributeError('Centroid homfold is not permitted to run with "-g" or "-t".')
ch_params += ' -g -1'
first_structures = run_centroid_homfold(fasta2predict, fasta_homologous_seqs, centroid_homfold_params=ch_params)
structures2return = [ch_struc for ch_struc in centroid_homfold_select_best(first_structures)]
BA_support.remove_one_file_with_try(first_structures)
return structures2return
def build_cm_model_rsearch(query_seq, path2selected_sim_array):
ml.debug(fname())
query_structure = rna_blast_analyze.BR_core.viennaRNA.RNAfold(
str(query_seq.seq))[0]
# remove any annotations from query:
qs_clean = deepcopy(query_seq)
qs_clean.annotations = dict()
qs_clean.letter_annotations = dict()
# query_structure = RNA.fold(str(analyzed_hits.query.seq))[0]
# build stockholm like file for use in cm mohdel build
st_like = StockholmAlig()
st_like.append(qs_clean)
st_like.column_annotations['SS_cons'] = query_structure
fds, stock_file = mkstemp(prefix='rba_', suffix='_30', dir=CONFIG.tmpdir)
with os.fdopen(fds, 'w') as f:
st_like.write_stockholm(f)
# run actual cmbuild
cm_model_file = run_cmbuild(
stock_file,
cmbuild_params='--rsearch {}'.format(path2selected_sim_array))
# cleanup
BA_support.remove_one_file_with_try(stock_file)
return cm_model_file
def centroid_homfold_fast(all_seqs, query, all_seqs_fasta, n, centroid_homfold_params, len_diff):
ml.debug(fname())
selected_seqs = centroid_homfold_fast_prep(all_seqs, query, n, len_diff)
ch, homologous_file = mkstemp(prefix='rba_', suffix='_74', dir=CONFIG.tmpdir)
with os.fdopen(ch, 'w') as h:
SeqIO.write(selected_seqs, h, 'fasta')
structures, _ = me_centroid_homfold(all_seqs_fasta, homologous_file, params=centroid_homfold_params)
BA_support.remove_one_file_with_try(homologous_file)
return structures
def extend_meta_core(analyzed_hits,
query,
args_inner,
all_short,
multi_query,
iteration,
ih_model,
timeout=None):
ml.debug(fname())
# update params if different config is requested
CONFIG.override(tools_paths(args_inner.config_file))
blast_args = deepcopy(args_inner)
locarna_args = deepcopy(args_inner)
b_all_short = deepcopy(all_short)
l_all_short = deepcopy(all_short)
if args_inner.repredict_file is None:
fd, repred_file = mkstemp(prefix='rba_',
suffix='_18',
dir=CONFIG.tmpdir)
os.close(fd)
else:
repred_file = args_inner.repredict_file
for i, args in enumerate([blast_args, locarna_args]):
args.prediction_method = []
args.pred_params = dict()
args.dump = None
args.pdf_out = None
args.pandas_dump = None
args.repredict_file = repred_file + str(i)
args.dev_pred = False
args.logfile = None
args.json = None
args.html = None
args.cm_file = ih_model
analyzed_hits_simple = deepcopy(analyzed_hits)
analyzed_hits_locarna = deepcopy(analyzed_hits)
analyzed_hits_simple, _, _, _ = extend_simple_core(analyzed_hits_simple,
query, blast_args,
b_all_short,
multi_query, iteration,
ih_model)
analyzed_hits_locarna, _, _, _ = extend_locarna_core(analyzed_hits_locarna,
query,
locarna_args,
l_all_short,
multi_query,
iteration,
ih_model,
timeout=timeout)
# add cmstat to query
analyzed_hits.query = analyzed_hits_simple.query
order_out = []
b_dict = {BA_support.get_hit_n(h): h for h in analyzed_hits_simple.hits}
l_dict = {BA_support.get_hit_n(h): h for h in analyzed_hits_locarna.hits}
ok_keys = sorted(set(b_dict.keys()) | set(l_dict.keys()))
for inum in ok_keys:
bh = b_dict.get(inum, None)
lh = l_dict.get(inum, None)
hits = [bh, lh]
# fallback to simple if locarna returned empty hit
# deal with the situation when both ways returned empty hits
filtered_hits = [h for h in hits if h is not None]
if len(filtered_hits) == 1:
msg = 'Only one extension method completed successfully for {}. ' \
'Choosing the successfully extended sequence to the output.'.format(filtered_hits[0].extension.id)
ml.info(msg)
if ml.getEffectiveLevel() < 20:
print(msg)
analyzed_hits.hits.append(filtered_hits[0])
continue
elif len(filtered_hits) == 0:
# append empty extension
analyzed_hits.hits_failed.append(lh)
continue
bit_scores = [
i.extension.annotations['cmstat']['bit_sc'] for i in hits
]
mb = max(bit_scores)
bit_index = [i for i, j in enumerate(bit_scores) if j == mb][0]
order_out.append(bit_index)
analyzed_hits.hits.append(hits[bit_index])
# build failed hits
b_dict_failed = {
BA_support.get_hit_n(h): h
for h in analyzed_hits_simple.hits_failed
}
l_dict_failed = {
BA_support.get_hit_n(h): h
for h in analyzed_hits_locarna.hits_failed
}
for inum in sorted(set(b_dict_failed) | set(l_dict_failed)):
if inum not in ok_keys:
if inum in b_dict_failed:
analyzed_hits.hits_failed.append(b_dict_failed[inum])
elif inum in l_dict_failed:
analyzed_hits.hits_failed.append(l_dict_failed[inum])
else:
raise KeyError(
"Failed to find inum key in failed extensions. This should not happen."
)
# build the repredict file here if needed
if args_inner.repredict_file:
b_repredict = BA_support.iter2file_name(blast_args.repredict_file,
multi_query, iteration)
l_repredict = BA_support.iter2file_name(blast_args.repredict_file,
multi_query, iteration)
o_repredict = BA_support.iter2file_name(args_inner.repredict_file,
multi_query, iteration)
with open(b_repredict,
'r') as barf, open(l_repredict,
'r') as larf, open(o_repredict,
'w') as reprf:
"""
please note that order of files to merge must be same as the order of methods in previous for cycle
ie same as the one in which order_out var is set
"""
bb = (barf, larf)
fl = bb[0].readline()
reprf.write(fl)
fl = bb[0].readline()
reprf.write(fl)
# dump first line of the other documents
[[i.readline() for _ in range(1)] for i in bb[1:]]
for o in order_out:
lll = [i.readline() for i in bb]
reprf.write(lll[o])
# recreate needed data from selected hits
homology_prediction = []
homol_seqs = []
for hit in analyzed_hits.hits:
homology_prediction.append(hit.hpred)
if hit.hpred:
homol_seqs.append(hit.extension)
# add default prediction if it is not present
if 'ss0' not in hit.extension.letter_annotations:
if 'sss' not in hit.extension.annotations:
hit.extension.anotations['sss'] = []
hit.extension.annotations['sss'] += ['ss0']
hit.extension.letter_annotations['ss0'] = '.' * len(
hit.extension.seq)
# recreate needed data from selected hits
homology_prediction = []
homol_seqs = []
for hit in analyzed_hits.hits:
homology_prediction.append(hit.hpred)
if hit.hpred:
homol_seqs.append(hit.extension)
# add default prediction if it is not present
if 'ss0' not in hit.extension.letter_annotations:
if 'sss' not in hit.extension.annotations:
hit.extension.anotations['sss'] = []
hit.extension.annotations['sss'] += ['ss0']
hit.extension.letter_annotations['ss0'] = '.' * len(
hit.extension.seq)
# remove description from hits and sources
for hit in analyzed_hits.hits:
hit.extension.description = ''
if args_inner.cm_file or args_inner.use_rfam:
cm_file_rfam_user = ih_model
else:
cm_file_rfam_user = None
BA_support.remove_one_file_with_try(ih_model)
return analyzed_hits, homology_prediction, homol_seqs, cm_file_rfam_user
def infer_homology(analyzed_hits,
args,
cm_model_file,
multi_query=False,
iteration=0):
"""
This is wrapper for infer homology methods. It deals with different options for generating CM.
:return:
"""
ml.info('Infering homology...')
ml.debug(fname())
bits, eval, loc_score, alig_length = hit_cons_characteristic(
analyzed_hits.hits)
# always run cmscan on rfam for informative reasons
# but use inferred CM only if --use_rfam was given
# if CM provided, also run inference but use provided file
# print explanation alongside this information
# find and extract cm model
# This code is moved to each extension method to allow fail-fast if model is found in RFAM
# cm_model_file, analyzed_hits = find_and_extract_cm_model(args, analyzed_hits)
# include query seq in fasta file to get relevant bit score
fd_f, fd_fasta = mkstemp(prefix='rba_', suffix='_28', dir=CONFIG.tmpdir)
with os.fdopen(fd_f, 'w') as f:
for seq in [analyzed_hits.query] + analyzed_hits.res_2_record_list():
f.write('>{}\n{}\n'.format(seq.id, str(seq.seq)))
cm_msa, cm_align_scores = run_cmalign_with_scores(fd_fasta,
cm_model_file,
threads=args.threads)
_add_rsearch_align_scores2anal_hits(analyzed_hits, cm_align_scores)
# remove first 1 (query) from the prediction scores
prediction = infer_hits_cm(cm_align_scores[1:].bit_sc)
# write scores to a table, compute it for all data and run some correlation statistics
if args.repredict_file:
# note that the first score is for the query and act as a benchmark here
cm_msa_conservation = alignment_sequence_conservation(cm_msa,
gap_chars='-.')
repredict_file = BA_support.iter2file_name(args.repredict_file,
multi_query, iteration)
with open(repredict_file, 'w') as f:
_print_table_for_corelation(f, cm_align_scores.seq_name[1:], bits,
eval, loc_score, alig_length,
cm_msa_conservation[1:],
cm_align_scores.bit_sc[1:],
cm_msa_conservation[0],
cm_align_scores.bit_sc[0])
BA_support.remove_one_file_with_try(fd_fasta)
selected_hits = [
hit.extension for b, hit in zip(prediction, analyzed_hits.hits) if b
]
if args.cm_file or args.use_rfam:
r_cm_file = cm_model_file
else:
r_cm_file = None
BA_support.remove_one_file_with_try(cm_model_file)
return prediction, selected_hits, r_cm_file
def locarna_worker(pack):
ml.debug(fname())
one_expanded_hit, query_seq, locarna_params, anchor_length = pack
locarna_file1 = locarna_file2 = loc_out_file = None
try:
# read the aligned segment and use it as anchors for locarna
# run locarna in local mode and put the query sequence with the extended sequence with the blast aligned
# segment as anchor
blast_entry = one_expanded_hit.annotations['blast'][1]
anchors = LocarnaAnchor(
to_rna(blast_entry.query),
blast_entry.match,
to_rna(blast_entry.sbjct),
anchor_length=anchor_length
)
if anchors.too_many_anchors:
ml.info('Too many anchors for {}. Can handle up to 520 distinct anchors.'.format(one_expanded_hit.id))
# extracted temp is my query
# access the locarna aligner directly
fd1, locarna_file1 = mkstemp(prefix='rba_', suffix='_20', dir=CONFIG.tmpdir)
with os.fdopen(fd1, 'w') as fp_locarna_file_1:
ql1, ql2 = anchors.anchor_whole_seq(str(query_seq), 'query')
write_clustal_like_file_with_anchors(fp_locarna_file_1,
'query',
str(query_seq),
(
('#A1', ql1.split()[0]),
('#A2', ql2.split()[0])
))
fd2, locarna_file2 = mkstemp(prefix='rba_', suffix='_21', dir=CONFIG.tmpdir)
with os.fdopen(fd2, 'w') as fp_locarna_file_2:
sl1, sl2 = anchors.anchor_whole_seq(str(one_expanded_hit.seq), 'subject')
write_clustal_like_file_with_anchors(fp_locarna_file_2,
one_expanded_hit.id,
str(one_expanded_hit.seq),
(
('#A1', sl1.split()[0]),
('#A2', sl2.split()[0])
))
loc_out_file = run_locarna(
locarna_file1,
locarna_file2,
locarna_params
)
# read locarna alignment
with open(loc_out_file, 'r') as f:
locarna_alig = parse_locarna_alignment(f)
if len(locarna_alig) != 2:
raise exceptions.SubseqMatchError('There must be 2 sequences in Locarna alignment.')
loc_rep = create_report_object_from_locarna(one_expanded_hit, locarna_alig)
return loc_rep
except exceptions.LocarnaException as e:
one_expanded_hit.annotations['msgs'] = [str(e), e.errors]
empty_hit = BA_support.Subsequences(one_expanded_hit)
return empty_hit
except (exceptions.SubseqMatchError, exceptions.ParsingError) as e:
one_expanded_hit.annotations['msgs'] = [str(e)]
empty_hit = BA_support.Subsequences(one_expanded_hit)
return empty_hit
except (TypeError, AttributeError, FileNotFoundError) as e:
one_expanded_hit.annotations['msgs'] = [str(e)]
empty_hit = BA_support.Subsequences(one_expanded_hit)
return empty_hit
finally:
for f in [locarna_file1, locarna_file2, loc_out_file]:
if f is not None:
BA_support.remove_one_file_with_try(f)
def repredict_structures_for_homol_seqs(
query,
seqs2predict_fasta,
threads=None,
prediction_method=None,
pred_method_params=None,
all_hits_list=None,
seqs2predict_list=None,
use_cm_file=None,
):
"""Run RNA structure prediction based on chosen method and parameters.
"""
default_sim_tr_perc = 90
default_score_tr = 0.0
query_max_len_diff = 0.1
try:
if 'default' == prediction_method:
# do nothing
return None, None, []
elif 'rfam-Rc' == prediction_method:
if use_cm_file is None:
msg = "No CM model. Can't use {}.".format(prediction_method)
ml.warning(msg)
return None, None, [msg]
else:
structures, exec_time = cmmodel_rnafold_c(
seqs2predict_fasta,
use_cm_file,
threads=threads,
params=pred_method_params.get(prediction_method, {}))
return structures, exec_time, []
elif 'rfam-centroid' == prediction_method:
# run cmscan if needed
# run cmfetch
# run cmemit -> homologous seqs
# run centroid_homfold
method_parameters = pred_method_params.get(prediction_method, {})
if use_cm_file is None:
msg = "No CM model. Can't use {}.".format(prediction_method)
ml.warning(msg)
return None, None, [msg]
else:
cep = method_parameters.get('cmemit', '')
if '-u' not in cep:
cep += ' -u'
if '-N' not in cep:
cep += ' -N {}'.format(method_parameters.get('n_seqs', 10))
hf_file = run_cmemit(use_cm_file, params=cep)
structures, exec_time = me_centroid_homfold(
seqs2predict_fasta, hf_file, params=method_parameters)
BA_support.remove_one_file_with_try(hf_file)
return structures, exec_time, []
elif 'rfam-sub' == prediction_method:
if use_cm_file is None:
msg = "No CM model. Can't use {}.".format(prediction_method)
ml.warning(msg)
return None, None, [msg]
else:
ref_structure = extract_ref_from_cm(use_cm_file)
structures, exec_time = rfam_subopt_pred(
seqs2predict_fasta,
ref_structure,
params=pred_method_params.get(prediction_method, None),
threads=threads,
)
return structures, exec_time, []
elif 'rnafold' == prediction_method:
structures, exec_time = rnafold_wrap_for_predict(
seqs2predict_fasta,
params=pred_method_params.get(prediction_method,
{}).get('RNAfold', ''))
return structures, exec_time, []
elif 'fq-sub' == prediction_method:
a, qf = mkstemp(prefix='rba_', suffix='_55', dir=CONFIG.tmpdir)
with os.fdopen(a, 'w') as fd:
fd.write('>query\n{}\n'.format(str(query.seq)))
structures, exec_time = subopt_fold_query(
seqs2predict_fasta,
qf,
params=pred_method_params.get(prediction_method, None),
threads=threads)
BA_support.remove_one_file_with_try(qf)
return structures, exec_time, []
elif 'C-A-sub' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, homologous_seqs, msgs = create_nr_trusted_hits_file_MSA_safe(
all_hits=all_hits_list,
query=query,
sim_threshold_percent=method_parameters.get(
'pred_sim_threshold', default_sim_tr_perc),
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del nr_homo_hits_file
f, homologous_sequence_file = mkstemp(prefix='rba_',
suffix='_64',
dir=CONFIG.tmpdir)
with os.fdopen(f, 'w') as fh:
SeqIO.write(homologous_seqs, fh, 'fasta')
structures, exec_time = subopt_fold_alifold(
seqs2predict_fasta,
homologous_sequence_file,
aligner='clustalo',
params=method_parameters,
threads=threads)
BA_support.remove_one_file_with_try(homologous_sequence_file)
del homologous_sequence_file
del homologous_seqs
return structures, exec_time, msgs
elif 'M-A-sub' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, homologous_seqs, msgs = create_nr_trusted_hits_file_MSA_safe(
all_hits=all_hits_list,
query=query,
sim_threshold_percent=method_parameters.get(
'pred_sim_threshold', default_sim_tr_perc),
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del nr_homo_hits_file
f, homologous_sequence_file = mkstemp(prefix='rba_',
suffix='_65',
dir=CONFIG.tmpdir)
with os.fdopen(f, 'w') as fh:
SeqIO.write(homologous_seqs, fh, 'fasta')
structures, exec_time = subopt_fold_alifold(
seqs2predict_fasta,
homologous_sequence_file,
aligner='muscle',
params=method_parameters,
threads=threads,
)
BA_support.remove_one_file_with_try(homologous_sequence_file)
del homologous_sequence_file
del homologous_seqs
return structures, exec_time, msgs
elif 'C-A-r-Rc' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, _, msgs = create_nr_trusted_hits_file_MSA_safe(
all_hits=all_hits_list,
query=query,
sim_threshold_percent=method_parameters.get(
'pred_sim_threshold', default_sim_tr_perc),
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
structures, exec_time = alifold_refold_prediction(
nr_homo_hits_file,
seqs2predict_fasta,
refold='refold_rnafoldc',
threads=threads,
params=method_parameters,
msa_alg='clustalo')
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del nr_homo_hits_file
return structures, exec_time, msgs
elif 'M-A-r-Rc' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, _, msgs = create_nr_trusted_hits_file_MSA_safe(
all_hits=all_hits_list,
query=query,
sim_threshold_percent=method_parameters.get(
'pred_sim_threshold', default_sim_tr_perc),
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
structures, exec_time = alifold_refold_prediction(
nr_homo_hits_file,
seqs2predict_fasta,
refold='refold_rnafoldc',
threads=threads,
params=method_parameters,
msa_alg='muscle')
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del nr_homo_hits_file
return structures, exec_time, msgs
elif 'C-A-U-r-Rc' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, _, msgs = create_nr_trusted_hits_file_MSA_safe(
all_hits=all_hits_list,
query=query,
sim_threshold_percent=method_parameters.get(
'pred_sim_threshold', default_sim_tr_perc),
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
structures, exec_time = alifold_refold_prediction(
nr_homo_hits_file,
seqs2predict_fasta,
refold='conserved_ss_rnafoldc',
threads=threads,
params=method_parameters,
msa_alg='clustalo')
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del nr_homo_hits_file
return structures, exec_time, msgs
elif 'M-A-U-r-Rc' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, _, msgs = create_nr_trusted_hits_file_MSA_safe(
all_hits=all_hits_list,
query=query,
sim_threshold_percent=method_parameters.get(
'pred_sim_threshold', default_sim_tr_perc),
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
structures, exec_time = alifold_refold_prediction(
nr_homo_hits_file,
seqs2predict_fasta,
refold='conserved_ss_rnafoldc',
threads=threads,
params=method_parameters,
msa_alg='muscle')
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del nr_homo_hits_file
return structures, exec_time, msgs
elif 'centroid' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, _, msgs = create_nr_homolog_hits_file_MSA_unsafe(
all_hits=all_hits_list,
query=query,
sim_threshold_percent=method_parameters.get(
'pred_sim_threshold', default_sim_tr_perc),
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
raw_structures, exec_time = me_centroid_homfold(
seqs2predict_fasta,
nr_homo_hits_file,
params=method_parameters)
# check noncanonical
if prediction_method in pred_method_params and pred_method_params[
prediction_method]:
allow_nc = pred_method_params[prediction_method].get(
'allow_noncanonical', False)
allow_lp = pred_method_params[prediction_method].get(
'allow_lonely_pairs', False)
else:
allow_nc = False
allow_lp = False
if not allow_nc:
for seq in raw_structures:
repstr = find_nc_and_remove(
str(seq.seq), structure=seq.letter_annotations['ss0'])
seq.letter_annotations['ss0'] = repstr
# check lonely basepairs
if not allow_lp:
for seq in raw_structures:
repstr = check_lonely_bp(seq.letter_annotations['ss0'])
seq.letter_annotations['ss0'] = repstr
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del nr_homo_hits_file
return raw_structures, exec_time, msgs
elif 'centroid-fast' == prediction_method:
method_parameters = pred_method_params.get(prediction_method, {})
if query.annotations['ambiguous']:
raise exceptions.AmbiguousQuerySequenceException
raw_structures, exec_time = centroid_homfold_fast(
all_seqs=all_hits_list,
query=query,
all_seqs_fasta=seqs2predict_fasta,
n=method_parameters.get('max_seqs_in_prediction', 10),
centroid_homfold_params=method_parameters,
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff))
# check noncanonical
if prediction_method in pred_method_params and pred_method_params[
prediction_method]:
allow_nc = pred_method_params[prediction_method].get(
'allow_noncanonical', False)
allow_lp = pred_method_params[prediction_method].get(
'allow_lonely_pairs', False)
else:
allow_nc = False
allow_lp = False
if not allow_nc:
for seq in raw_structures:
repstr = find_nc_and_remove(
str(seq.seq), structure=seq.letter_annotations['ss0'])
seq.letter_annotations['ss0'] = repstr
# check lonely basepairs
if not allow_lp:
for seq in raw_structures:
repstr = check_lonely_bp(seq.letter_annotations['ss0'])
seq.letter_annotations['ss0'] = repstr
return raw_structures, exec_time, []
elif 'TurboFold' == prediction_method:
# set arbitrary sim_threshold_percent to 100, because we want to remove only identical sequences from prediction
# with TurboFold. The structure of redundant sequences will be set according to the one in prediction
all_hits_filtered = BA_support.filter_ambiguous_seqs_from_list(
all_hits_list)
seqs2predict_filtered = BA_support.filter_ambiguous_seqs_from_list(
seqs2predict_list)
if len(seqs2predict_list) != len(seqs2predict_filtered):
ml.warning(
'Some sequences contain ambiguous bases - they will not be predicted.'
)
if query.annotations['ambiguous']:
raise exceptions.AmbiguousQuerySequenceException()
method_parameters = pred_method_params.get(prediction_method, {})
nr_homo_hits_file, _, msgs = create_nr_homolog_hits_file_MSA_unsafe(
all_hits=all_hits_filtered,
query=query,
sim_threshold_percent=100,
cmscore_tr=method_parameters.get('cmscore_tr',
default_score_tr),
cm_threshold_percent=method_parameters.get(
'cmscore_percent', None),
len_diff=method_parameters.get('query_max_len_diff',
query_max_len_diff),
)
with open(nr_homo_hits_file, 'r') as nrf:
nr_homo_hits = [
seq for seq in SeqIO.parse(nrf, format='fasta')
]
nh = sha1()
nh.update(str(sorted(method_parameters.items())).encode())
nh_str = nh.hexdigest()
structures_t, exec_time = turbofold_with_homologous(
all_sequences=seqs2predict_filtered,
nr_homologous=nr_homo_hits,
params=method_parameters.get('TurboFold', {}),
n=method_parameters.get('max_seqs_in_prediction', 3),
cpu=threads,
pkey=prediction_method,
sha1val=nh_str,
)
structures = BA_support.rebuild_structures_output_from_pred(
seqs2predict_list, structures_t)
BA_support.remove_one_file_with_try(nr_homo_hits_file)
del structures_t
del nr_homo_hits
del nr_homo_hits_file
return structures, exec_time, msgs
elif 'Turbo-fast' == prediction_method:
if query.annotations['ambiguous']:
raise exceptions.AmbiguousQuerySequenceException()
nh = sha1()
nh.update(
str(
sorted(
pred_method_params.get(prediction_method,
{}).items())).encode())
nh_str = nh.hexdigest()
structures_t, exec_time = turbofold_fast(
all_seqs=all_hits_list,
seqs2predict=seqs2predict_list,
query=query,
cpu=threads,
n=pred_method_params.get(prediction_method,
{}).get('max_seqs_in_prediction', 3),
turbofold_params=pred_method_params.get(prediction_method,
{}).get(
'TurboFold', {}),
len_diff=pred_method_params.get(prediction_method, {}).get(
'query_max_len_diff', query_max_len_diff),
pkey=prediction_method,
sha1val=nh_str,
)
structures = BA_support.rebuild_structures_output_from_pred(
seqs2predict_list, structures_t)
del structures_t
return structures, exec_time, []
except exceptions.NoHomologousSequenceException:
msg = nonhomseqwarn(prediction_method)
return None, None, [msg]
except exceptions.AmbiguousQuerySequenceException:
msgfail = "Query sequence contains ambiguous characters. Can't use {}.".format(
prediction_method)
ml.warning(msgfail)
return None, None, [msgfail]
except exceptions.SubprocessException as e:
msg = "{} can't be used. Error message follows: {} \n{}".format(
prediction_method, str(e), e.errors)
ml.error(msg)
return None, None, [str(e)]
except Exception as e:
ml.error("{} can't be used. Error message follows: \n{}.".format(
prediction_method, str(e)))
return None, None, [str(e)]
assert False, "Should not reach here (bad prediction method name)."
def wrapped_ending_with_prediction(
args_inner,
analyzed_hits,
pred_method=None,
method_params=None,
used_cm_file=None,
multi_query=False,
iteration=0,
):
"""
wrapper for prediction of secondary structures
:param args_inner: Namespace of input arguments
:param analyzed_hits: BlastSearchRecompute object
:param pred_method:
:param method_params:
:param used_cm_file: cmfile if cmfile is known (user given or computed)
:return:
"""
ml.debug(fname())
exec_time = {}
msg = 'Entering structure prediction..'
if ml.level < 21:
ml.info(msg)
else:
print(msg)
ml.info(msg)
if pred_method is None:
pred_method = args_inner.prediction_method
if isinstance(pred_method, str):
pred_method = (pred_method, )
if method_params is None:
method_params = args_inner.pred_params
# ======= filter if needed =======
# do the filtering based on e-val or bitscore
# homologous hits still gets used for prediction
# annotate ambiguous bases
query = BA_support.annotate_ambiguos_base(analyzed_hits.query)
# copy the list before filtering
all_hits_list = [i.extension for i in analyzed_hits.get_all_hits()]
if args_inner.filter_by_eval is not None:
hits2predict = filter_by_eval(analyzed_hits.get_all_hits(),
BA_support.blast_hit_getter_from_subseq,
args_inner.filter_by_eval)
_hits = HitList()
for h in hits2predict:
_hits.append(h)
analyzed_hits.hits = _hits
elif args_inner.filter_by_bitscore is not None:
hits2predict = filter_by_bits(analyzed_hits.get_all_hits(),
BA_support.blast_hit_getter_from_subseq,
args_inner.filter_by_bitscore)
_hits = HitList()
for h in hits2predict:
_hits.append(h)
analyzed_hits.hits = _hits
else:
analyzed_hits.hits = analyzed_hits.get_all_hits()
# if used_cm_file is provided do not override it with CM from RFAM
# if use_rfam flag was given, then used_cm_file is already the best_matching model
# if analyzed_hits.best_matching_model is None - then we could not find the best matching model in RFAM
# and the rfam based methods should fail (i.e. not predict anything)
delete_cm = False
if used_cm_file is None and analyzed_hits.best_matching_model is not None:
rfam = RfamInfo()
used_cm_file = run_cmfetch(
rfam.file_path, analyzed_hits.best_matching_model['target_name'])
delete_cm = True
fd, seqs2predict_fasta = mkstemp(prefix='rba_',
suffix='_83',
dir=CONFIG.tmpdir)
with os.fdopen(fd, 'w') as fah:
for hit in analyzed_hits.hits:
if len(hit.extension.seq) == 0:
continue
fah.write('>{}\n{}\n'.format(hit.extension.id,
str(hit.extension.seq)))
if not isinstance(method_params, dict):
raise Exception('prediction method parameters must be python dict')
# prediction methods present in analyzed_hits
# which might be loaded from intermediate file
# check if structures of a method are predicted for all required hit
# check if the prediction parameters of such method were same
prediction_msgs = []
# compute prediction methods which were not computed
for pkey in set(pred_method):
# add sha1 hashes
nh = sha1()
nh.update(str(sorted(method_params.get(pkey, {}).items())).encode())
current_hash = nh.hexdigest()
if all(pkey in h.extension.letter_annotations for h in analyzed_hits.hits) and \
len(
{
h.extension.annotations.get('sha1', {}).get(pkey, None) for h in analyzed_hits.hits
} | {current_hash, }
) == 1:
msg_skip = 'All structures already computed for {}. Skipping...'.format(
pkey)
ml.info(msg_skip)
if ml.level > 20:
print(msg_skip, flush=True)
continue
msg_run = 'Running: {}...'.format(pkey)
ml.info(msg_run)
if ml.level > 20:
print(msg_run, flush=True)
structures, etime, msgs = repredict_structures_for_homol_seqs(
query,
seqs2predict_fasta,
args_inner.threads,
prediction_method=pkey,
pred_method_params=method_params,
all_hits_list=all_hits_list,
seqs2predict_list=[i.extension for i in analyzed_hits.hits],
use_cm_file=used_cm_file,
)
exec_time[pkey] = etime
if structures is None:
msg = 'Structures not predicted with {} method'.format(pkey)
ml.info(msg)
if ml.level > 20:
print('STATUS: ' + msg)
else:
for i, hit in enumerate(analyzed_hits.hits):
assert str(hit.extension.seq) == str(structures[i].seq)
hit.extension.annotations['sss'] += [pkey]
hit.extension.annotations['msgs'] += structures[
i].annotations.get('msgs', [])
# expects "predicted" in annotations - for now, if not given, default is True, as not all prediction
# methods implement "predicted" in their output
if structures[i].annotations.get('predicted', True):
hit.extension.letter_annotations[pkey] = structures[
i].letter_annotations['ss0']
if 'sha1' not in hit.extension.annotations:
hit.extension.annotations['sha1'] = dict()
hit.extension.annotations['sha1'][pkey] = current_hash
try:
del hit.extension.letter_annotations['ss0']
except KeyError:
pass
try:
hit.extension.annotations['sss'].remove('ss0')
except ValueError:
pass
analyzed_hits.update_hit_stuctures()
# check if msgs are not empty
if msgs:
prediction_msgs.append('{}: {}'.format(pkey, '\n'.join(msgs)))
analyzed_hits.msgs = prediction_msgs
with open(args_inner.blast_in + '.r-' + args_inner.sha1[:10],
'r+') as f:
all_saved_data = json.load(f)
all_saved_data[iteration] = blastsearchrecompute2dict(
analyzed_hits)
f.seek(0)
f.truncate()
json.dump(all_saved_data, f, indent=2)
# remove structures predicted by different methods (which might be saved from previous computation)
for hit in analyzed_hits.hits:
for pkey in set(hit.extension.letter_annotations.keys()):
if pkey not in pred_method:
del hit.extension.letter_annotations[pkey]
try:
hit.extension.annotations['sss'].remove(pkey)
except ValueError:
pass
BA_support.remove_one_file_with_try(seqs2predict_fasta)
if delete_cm:
BA_support.remove_one_file_with_try(used_cm_file)
add_loc_to_description(analyzed_hits)
# write html if requested
if args_inner.html:
html_file = iter2file_name(args_inner.html, multi_query, iteration)
ml.info('Writing html to {}.'.format(html_file))
with open(html_file, 'wb') as h:
h.write(write_html_output(analyzed_hits))
# write csv file if requested
if args_inner.csv:
csv_file = iter2file_name(args_inner.csv, multi_query, iteration)
ml.info('Writing csv to {}.'.format(csv_file))
analyzed_hits.to_csv(csv_file)
# replace with json
if args_inner.json:
json_file = iter2file_name(args_inner.json, multi_query, iteration)
ml.info('Writing json to {}.'.format(json_file))
j_obj = json.dumps(blastsearchrecompute2dict(analyzed_hits), indent=2)
if getattr(args_inner, 'zip_json', False):
with open(json_file + '.gz', 'wb') as ff:
ff.write(gzip.compress(j_obj.encode()))
else:
with open(json_file, 'w') as ff:
ff.write(j_obj)
if args_inner.pandas_dump:
pickle_file = iter2file_name(args_inner.pandas_dump, multi_query,
iteration)
ml.info('Writing pandas pickle to {}.'.format(pickle_file))
pandas.to_pickle(analyzed_hits.pandas, pickle_file)
if args_inner.dump:
dump_file = iter2file_name(args_inner.dump, multi_query, iteration)
ml.info('Writing dump files base: {}.'.format(dump_file))
with open(dump_file, 'wb') as pp:
pickle.dump(analyzed_hits, pp, pickle.HIGHEST_PROTOCOL)
with open(dump_file + '.time_dump', 'wb') as pp:
pickle.dump(exec_time, pp, pickle.HIGHEST_PROTOCOL)
return analyzed_hits
def lunch_computation(args_inner, shared_list=None):
ml.debug(fname())
if not shared_list:
shared_list = []
# update params if different config is requested
CONFIG.override(tools_paths(args_inner.config_file))
p_blast = BA_support.blast_in(args_inner.blast_in, b=args_inner.b_type)
query_seqs = [i for i in SeqIO.parse(args_inner.blast_query, 'fasta')]
if len(p_blast) != len(query_seqs):
ml.error(
'Number of query sequences in provided BLAST output file ({}) does not match number of query sequences'
' in query FASTA file ({}).'.format(len(p_blast), len(query_seqs)))
sys.exit(1)
# check if BLAST does not contain unexpected sequence characters
validate_args.check_blast(p_blast)
# create list of correct length if needed
all_saved_data = [None] * len(query_seqs)
saved_file = '{}.r-{}'.format(args_inner.blast_in, args_inner.sha1[:10])
with open(saved_file, 'r+') as f:
_saved = json.load(f)
if _saved is None:
f.seek(0)
f.truncate()
json.dump(all_saved_data, f)
else:
msg = "Loading backup data."
print('STATUS: ' + msg)
ml.info(msg + ' file: ' + saved_file)
all_saved_data = _saved
for saved_data in all_saved_data:
# we can have partially computed data
if saved_data is None:
continue
if saved_data['args']['sha1'] != args_inner.sha1:
msg = "Input argument hash does not match the saved argument hash. "
if saved_data['args']['sha1'][:10] == args_inner.sha1[:10]:
msg += "This is because of truncating hashes to first 10 characters. "
msg += "Please remove the '{}' file.".format(
saved_file)
ml.error(msg)
sys.exit(1)
else:
msg += "Please remove the '{}' file.".format(
saved_file)
sys.exit(1)
if len(p_blast) > 1:
multi_query = True
else:
multi_query = False
# this is done for each query
ml_out_line = []
all_analyzed = []
for iteration, (bhp, query, saved_data) in enumerate(
zip(p_blast, query_seqs, all_saved_data)):
if saved_data is None:
print('STATUS: processing query: {}'.format(query.id))
validate_args.verify_query_blast(blast=bhp, query=query)
analyzed_hits = BlastSearchRecompute(args_inner, query, iteration)
analyzed_hits.multi_query = multi_query
# run cm model build
# allows to fail fast if rfam was selected and we dont find the model
ih_model, analyzed_hits = find_and_extract_cm_model(
args_inner, analyzed_hits)
# select all
all_blast_hits = BA_support.blast_hsps2list(bhp)
if len(all_blast_hits) == 0:
ml.error('No hits found in {} - {}. Nothing to do.'.format(
args_inner.blast_in, bhp.query))
continue
# filter if needed
if args_inner.filter_by_eval is not None:
tmp = filter_by_eval(all_blast_hits,
BA_support.blast_hit_getter_from_hits,
args_inner.filter_by_eval)
if len(tmp) == 0 and len(all_blast_hits) != 0:
ml.error(
'The requested filter removed all BLAST hits {} - {}. Nothing to do.'
.format(args_inner.blast_in, bhp.query))
continue
elif args_inner.filter_by_bitscore is not None:
tmp = filter_by_bits(all_blast_hits,
BA_support.blast_hit_getter_from_hits,
args_inner.filter_by_bitscore)
if len(tmp) == 0 and len(all_blast_hits) != 0:
ml.error(
'The requested filter removed all BLAST hits {} - {}. Nothing to do.'
.format(args_inner.blast_in, bhp.query))
continue
all_short = all_blast_hits
# now this is different for each mode
if args_inner.mode == 'simple':
analyzed_hits, homology_prediction, homol_seqs, cm_file_rfam_user = extend_simple_core(
analyzed_hits, query, args_inner, all_short, multi_query,
iteration, ih_model)
elif args_inner.mode == 'locarna':
analyzed_hits, homology_prediction, homol_seqs, cm_file_rfam_user = extend_locarna_core(
analyzed_hits, query, args_inner, all_short, multi_query,
iteration, ih_model)
elif args_inner.mode == 'meta':
analyzed_hits, homology_prediction, homol_seqs, cm_file_rfam_user = extend_meta_core(
analyzed_hits, query, args_inner, all_short, multi_query,
iteration, ih_model)
else:
raise ValueError(
'Unknown option - should be cached by argparse.')
if len(analyzed_hits.hits) == 0:
ml.error(
"Extension failed for all sequences. Please see the error message. You can also try '--mode simple'."
)
sys.exit(1)
analyzed_hits.copy_hits()
with open(args_inner.blast_in + '.r-' + args_inner.sha1[:10],
'r+') as f:
all_saved_data = json.load(f)
all_saved_data[iteration] = blastsearchrecompute2dict(
analyzed_hits)
f.seek(0)
f.truncate()
json.dump(all_saved_data, f, indent=2)
else:
print(
'STATUS: extended sequences loaded from backup file for query {}'
.format(query.id))
analyzed_hits = blastsearchrecomputefromdict(saved_data)
# overwrite the saved args with current
# this will update used prediction methods and other non essential stuff
analyzed_hits.args = args_inner
if analyzed_hits.args.cm_file:
cm_file_rfam_user = analyzed_hits.args.cm_file
else:
cm_file_rfam_user = None
all_analyzed.append(analyzed_hits)
# write all hits to fasta
fda, all_hits_fasta = mkstemp(prefix='rba_',
suffix='_22',
dir=CONFIG.tmpdir)
os.close(fda)
analyzed_hits.write_results_fasta(all_hits_fasta)
out_line = []
# multiple prediction params
if args_inner.dev_pred:
dp_list = []
# acomodate more dev pred outputs
dpfile = None
if getattr(args_inner, 'dump', False):
dpfile = args_inner.dump.strip('dump')
if getattr(args_inner, 'pandas_dump', False):
dpfile = args_inner.pandas_dump.strip('pandas_dump')
if getattr(args_inner, 'json', False):
dpfile = args_inner.json.strip('json')
# optimization so the rfam cm file is used only once
if cm_file_rfam_user is None and 'rfam' in ''.join(
args_inner.prediction_method):
best_model = get_cm_model(args_inner.blast_query,
threads=args_inner.threads)
rfam = RfamInfo()
cm_file_rfam_user = run_cmfetch(rfam.file_path, best_model)
for method in args_inner.prediction_method:
# cycle the prediction method settings
# get set of params for each preditcion
selected_pred_params = [
kk for kk in args_inner.pred_params if method in kk
]
shuffle(selected_pred_params)
# for method_params in args_inner.pred_params:
for i, method_params in enumerate(selected_pred_params):
ah = deepcopy(analyzed_hits)
random_flag = BA_support.generate_random_name(
8, shared_list)
shared_list.append(random_flag)
pname = re.sub(' ', '', str(method))
flag = '|pred_params|' + random_flag
# rebuild the args only with actualy used prediction settings
ah.args.prediction_method = method
ah.args.pred_params = method_params
if getattr(args_inner, 'dump', False):
spa = args_inner.dump.split('.')
ah.args.dump = '.'.join(
spa[:-1]) + flag + '.' + spa[-1]
if getattr(args_inner, 'pandas_dump', False):
spa = args_inner.pandas_dump.split('.')
ah.args.pandas_dump = '.'.join(
spa[:-1]) + flag + '.' + spa[-1]
if getattr(args_inner, 'pdf_out', False):
spa = args_inner.pdf_out.split('.')
ah.args.pdf_out = '.'.join(
spa[:-1]) + flag + '.' + spa[-1]
if getattr(args_inner, 'json', False):
spa = args_inner.json.split('.')
ah.args.json = '.'.join(
spa[:-1]) + flag + '.' + spa[-1]
wrapped_ending_with_prediction(
args_inner=ah.args,
analyzed_hits=ah,
pred_method=method,
method_params=method_params,
used_cm_file=cm_file_rfam_user,
multi_query=multi_query,
iteration=iteration,
)
success = True
out_line.append(to_tab_delim_line_simple(ah.args))
dp_list.append((i, method_params, success, flag, pname,
random_flag, args_inner.pred_params))
if dpfile is not None:
with open(dpfile + 'devPredRep', 'wb') as devf:
pickle.dump(dp_list, devf)
else:
wrapped_ending_with_prediction(
args_inner=args_inner,
analyzed_hits=analyzed_hits,
used_cm_file=cm_file_rfam_user,
multi_query=multi_query,
iteration=iteration,
)
out_line.append(to_tab_delim_line_simple(args_inner))
ml_out_line.append('\n'.join(out_line))
if cm_file_rfam_user is not None and os.path.exists(cm_file_rfam_user):
BA_support.remove_one_file_with_try(cm_file_rfam_user)
BA_support.remove_one_file_with_try(all_hits_fasta)
return '\n'.join(ml_out_line), all_analyzed