def run_cmalign_with_scores(fasta_file, cm_file, threads=None): fd_sfile, cm_sfile_path = mkstemp(prefix='rba_', suffix='_29', dir=CONFIG.tmpdir) os.close(fd_sfile) if threads: cm_params = '--notrunc --cpu {} --sfile {}'.format( threads, cm_sfile_path) else: cm_params = '--notrunc --sfile {}'.format(cm_sfile_path) cm_msa_file = run_cmalign_on_fasta(fasta_file, cm_file, cmalign_params=cm_params) cm_msa = read_st(cm_msa_file) # combine the eval and cm_msa_conservation_score # the cmalign scores somehow, look into the scoring if those scores are accessible, maybe they are far better then # my made up msa_conservation # there is - by option --sfile cm_align_scores = read_cmalign_sfile(cm_sfile_path) # the bit score can be probably directly comparable with blast bit score # i can also leverage the fact, that the badly aligned sequences with cmalign have negative bitscore # so my score can be cm_align_scores.index = range(len(cm_align_scores.index)) BA_support.remove_files_with_try([cm_sfile_path, cm_msa_file]) return cm_msa, cm_align_scores
def refold_stockholm(stockholm_alig, consensus_structure): """ compute refold.pl from Vienna RNA package :param stockholm_alig: :param consensus_structure: :return: """ ml.debug(fname()) # convert to clustal alignment fd, clust_tempfile = mkstemp(prefix='rba_', suffix='_23', dir=CONFIG.tmpdir) with os.fdopen(fd, 'w') as f: stockholm_alig.write_clustal(f) # write fake alifold output with given consensus structure fd, alif_fake_file = mkstemp(prefix='rba_', suffix='_24', dir=CONFIG.tmpdir) with os.fdopen(fd, 'w') as f: # the consensus sequence in alifold file is really not used for anything f.write('A'*len(consensus_structure) + '\n') f.write(consensus_structure + '\n') # compute refold # refold_path = locate_refold() refold_constrained_file = compute_refold(clust_tempfile, alif_fake_file) parsed_seqs = [] with open(refold_constrained_file, 'r') as f: # read the file for seq in BA_support.parse_seq_str(f): parsed_seqs.append(seq) # cleanup BA_support.remove_files_with_try([clust_tempfile, alif_fake_file, refold_constrained_file]) return parsed_seqs
def build_cm_model_rsearch(query_seq, path2selected_sim_array): ml.debug(fname()) query_structure = rna_blast_analyze.BR_core.viennaRNA.RNAfold( str(query_seq.seq))[0] # remove any annotations from query: qs_clean = deepcopy(query_seq) qs_clean.annotations = dict() qs_clean.letter_annotations = dict() # query_structure = RNA.fold(str(analyzed_hits.query.seq))[0] # build stockholm like file for use in cm mohdel build st_like = StockholmAlig() st_like.append(qs_clean) st_like.column_annotations['SS_cons'] = query_structure fds, stock_file = mkstemp(prefix='rba_', suffix='_30', dir=CONFIG.tmpdir) with os.fdopen(fds, 'w') as f: st_like.write_stockholm(f) # run actual cmbuild cm_model_file = run_cmbuild( stock_file, cmbuild_params='--rsearch {}'.format(path2selected_sim_array)) # cleanup BA_support.remove_one_file_with_try(stock_file) return cm_model_file
def me_centroid_homfold(fasta2predict, fasta_homologous_seqs, params=None): """ run centroid_homefold several times and vary -g parameter, to predict the best possible structure :param fasta2predict: :param fasta_homologous_seqs: :param params: :return: """ # first run centroid homefold for several stages of g (-1) # find the most stable structure value of g # structure of output if params is None: params = dict() ch_params = '' if params and ('centroid_homfold' in params) and params['centroid_homfold']: ch_params += params['centroid_homfold'] if '-g ' in ch_params and '-g -1' not in ch_params or '-t ' in params: print("We only allow to run centroid homfold in automatic mode where the structure is predicted with multiple" " weights and then best scoring structure is selected, threshold's are also forbidden as it implies -g.") raise AttributeError('Centroid homfold is not permitted to run with "-g" or "-t".') ch_params += ' -g -1' first_structures = run_centroid_homfold(fasta2predict, fasta_homologous_seqs, centroid_homfold_params=ch_params) structures2return = [ch_struc for ch_struc in centroid_homfold_select_best(first_structures)] BA_support.remove_one_file_with_try(first_structures) return structures2return
def create_report_object_from_locarna(exp_hit, locarna_alig): """ create object which will be appended to BlastSearchRecompute class This needs to be Subsequences object :param exp_hit: :param locarna_alig: :return: """ ml.debug(fname()) # chop alignment by seq query_ind = [i for i, j in enumerate(locarna_alig) if j.id == 'query'] if len(query_ind) != 1: raise exceptions.SubseqMatchError('Got multiple hits with id "query" in the Locarna alignment.') trimmed_locarna_alig = trim_alignment_by_sequence( locarna_alig, str(locarna_alig[query_ind[0]].seq), structure_annotation='SS_cons' ) aligned_subsequence = BA_support.select_analyzed_aligned_hit(trimmed_locarna_alig, exp_hit.id) # add annotations from exp hit aligned_subsequence.annotations = exp_hit.annotations aligned_subsequence.name = exp_hit.name # also add annotations from locarna, mainly score aligned_subsequence.annotations.update(locarna_alig.annotations) # get the structure # by refold refold_structures = refold_stockholm(trimmed_locarna_alig, trimmed_locarna_alig.column_annotations['SS_cons']) # select refold structure for my seq seq_refold_structure = _select_refold_structure(refold_structures, exp_hit.id) aligned_subsequence.letter_annotations['ss0'] = seq_refold_structure.letter_annotations['ss0'] aligned_subsequence.annotations['sss'] = ['ss0'] # prepare seq_record for subsequences aligned_subsequence.description = '' hit = BA_support.Subsequences(exp_hit) hit.extension = aligned_subsequence # find the matching sequence pos_match = re.search(str(aligned_subsequence.seq), str(exp_hit.seq), flags=re.IGNORECASE) if not pos_match: raise exceptions.SubseqMatchError( 'Aligned portion of subject sequence in Locarna alignment was not found in parent sequence.' ) hit.best_start, hit.best_end = compute_true_location_locarna(hit, pos_match) return hit
def centroid_homfold_fast(all_seqs, query, all_seqs_fasta, n, centroid_homfold_params, len_diff): ml.debug(fname()) selected_seqs = centroid_homfold_fast_prep(all_seqs, query, n, len_diff) ch, homologous_file = mkstemp(prefix='rba_', suffix='_74', dir=CONFIG.tmpdir) with os.fdopen(ch, 'w') as h: SeqIO.write(selected_seqs, h, 'fasta') structures, _ = me_centroid_homfold(all_seqs_fasta, homologous_file, params=centroid_homfold_params) BA_support.remove_one_file_with_try(homologous_file) return structures
def _prepare_set_n(seq, nr_seqs, n): if len(nr_seqs) - n + 1 < 0: return BA_support.non_redundant_seqs([seq] + nr_seqs) i = 0 while i != len(nr_seqs) - n + 1: seqs = [seq] + nr_seqs[:n - 1 + i] n_nr_seqs = BA_support.non_redundant_seqs(seqs) if len(n_nr_seqs) == n: return n_nr_seqs i += 1 return BA_support.non_redundant_seqs([seq] + nr_seqs)
def create_nr_homolog_hits_file_MSA_unsafe(sim_threshold_percent=None, all_hits=None, query=None, cmscore_tr=0.0, cm_threshold_percent=None, len_diff=0.1): """ create non redundant homologous hits file """ ml.debug(fname()) dist_table, homologous_seqs, msgs = _trusted_hits_selection_wrapper( all_hits, query, cmscore_tr, cm_threshold_percent, len_diff_=len_diff) if dist_table.size == 0: nr_homolog_hits = [query] else: # normal execution to_include = BA_support.select_sequences_from_similarity_rec( dist_table, sim_threshold_percent=sim_threshold_percent) nr_homolog_hits = [homologous_seqs[i] for i in to_include] fd_h, nr_homo_hits_file = mkstemp(prefix='rba_', suffix='_59', dir=CONFIG.tmpdir) with os.fdopen(fd_h, 'w') as f: SeqIO.write(nr_homolog_hits, f, 'fasta') return nr_homo_hits_file, homologous_seqs, msgs
def centroid_homfold_fast_prep(all_seqs, query, n, len_diff): ml.debug(fname()) assert n >= 1, "Number of sequences for centroid-fast must be greater then 0." if query.annotations['ambiguous']: msgfail = "Query sequence contains ambiguous characters. Can't use centroid-fast." ml.error(msgfail) raise AmbiguousQuerySequenceException(msgfail) nr_na_ld = BA_support.sel_seq_simple(all_seqs, query, len_diff) nr_na_ld_n = nr_na_ld[:int(n)] return nr_na_ld_n
def _parse_centroid_homefold_output_file(file): with open(file, 'r') as f: for sr in BA_support.parse_one_rec_in_multiline_structure(f): cf = sr.strip().splitlines() cfr = SeqRecord(Seq(cf[1]), id=cf[0]) cfr.annotations['sss'] = [] for i, ll in enumerate(cf[2:]): structure, ann = ll.split() cfr.letter_annotations['ss' + str(i)] = structure cfr.annotations['ss' + str(i)] = ann cfr.annotations['sss'].append('ss' + str(i)) yield cfr
def _turbofold_worker(sequences, params, timeout=None): env = os.environ.copy() if 'DATAPATH' not in env: env['DATAPATH'] = CONFIG.rnastructure_datapath tmpdir, con_file, output_structure_files = write_turbofold_confile( input_sequences=sequences, turbofold_params=params, ) # run without prediction progress reporting output with TemporaryFile(mode='w+', encoding='utf-8') as tmp: cmd = ['{}TurboFold'.format(CONFIG.turbofold_path), con_file] ml.debug(cmd) r = call(cmd, env=env, stdout=tmp, stderr=tmp, timeout=timeout) if r: msgfail = 'Call to TurboFold failed, cmd below:' ml.info(msgfail) ml.info(cmd) tmp.seek(0) raise exceptions.TurboFoldException(msgfail, tmp.read()) # now convert ct files produced by TurboFold new_structures = [] for out_str_file, orig_seq in zip(output_structure_files, sequences): o = open(out_str_file, 'r') seq_with_pred_str = BA_support.ct2db(o, energy_txt='ENERGY') o.close() assert str(seq_with_pred_str[0].seq) == str(orig_seq.seq), \ 'Input and output sequences of TurboFold call does not match.\n{}\n{}'.format( str(seq_with_pred_str[0].seq), str(orig_seq.seq) ) new_structures.append( SeqRecord(orig_seq.seq, id=orig_seq.id, annotations={'sss': ['ss0']}, letter_annotations={ 'ss0': seq_with_pred_str[0].letter_annotations['ss0'] })) rmtree(tmpdir) return new_structures
def create_blast_only_report_object(exp_hit, query_len): # init new Subsequences object # here the object source shadows the final hit hit = BA_support.Subsequences(exp_hit) # init new SeqRecord object ns = deepcopy(exp_hit) ann = ns.annotations tss = ann['trimmed_ss'] tse = ann['trimmed_se'] tes = ann['trimmed_es'] tee = ann['trimmed_ee'] ns.letter_annotations['ss0'] = '.' * len(ns.seq) ns.annotations['sss'] = ['ss0'] ns.description = '' hit.extension = ns bl = ns.annotations['blast'][1] if bl.sbjct_start < bl.sbjct_end: bls = bl.sbjct_start - bl.query_start + 1 ble = bl.sbjct_end + (query_len - bl.query_end) elif bl.sbjct_end < bl.sbjct_start: bls = bl.sbjct_end - (query_len - bl.query_end) ble = bl.sbjct_start + bl.query_start - 1 else: raise exceptions.UnknownStrand( "Can't determine HSP strand (sbjct_start appears equal to sbjct_end)" ) # if whole subject sequence too short, this assertion will fail if tss or tse or tes or tee: msg = 'STATUS: Skipping sequence check ({}) - subject sequence too short.'.format( ns.id) ml.info(msg) else: assert len(ns.seq) == abs(bls - ble) + 1 assert bls == ns.annotations['extended_start'] assert ble == ns.annotations['extended_end'] hit.best_start, hit.best_end = compute_true_location_se(hit, query_len) return hit
def turbofold_fast(all_seqs, seqs2predict, query, cpu, n, turbofold_params, len_diff, pkey, sha1val): """Run Turbo-fast prediction - ambiguous sequences cannot be predicted with TurboFold - do not predict sequence twice - seqs2predict must be subset of all_seqs """ ml.debug(fname()) # this is backup (in rboAnalyzer this is handled on higher level) if query.annotations['ambiguous']: msgfail = "Query sequence contains ambiguous characters. Can't use Turbo-fast." ml.error(msgfail) raise exceptions.AmbiguousQuerySequenceException(msgfail) nr_na_ld = BA_support.sel_seq_simple(all_seqs, query, len_diff) return turbofold_ext_nr_fast(seqs2predict, nr_na_ld, cpu, n, turbofold_params, pkey, sha1val)
def extend_meta_core(analyzed_hits, query, args_inner, all_short, multi_query, iteration, ih_model, timeout=None): ml.debug(fname()) # update params if different config is requested CONFIG.override(tools_paths(args_inner.config_file)) blast_args = deepcopy(args_inner) locarna_args = deepcopy(args_inner) b_all_short = deepcopy(all_short) l_all_short = deepcopy(all_short) if args_inner.repredict_file is None: fd, repred_file = mkstemp(prefix='rba_', suffix='_18', dir=CONFIG.tmpdir) os.close(fd) else: repred_file = args_inner.repredict_file for i, args in enumerate([blast_args, locarna_args]): args.prediction_method = [] args.pred_params = dict() args.dump = None args.pdf_out = None args.pandas_dump = None args.repredict_file = repred_file + str(i) args.dev_pred = False args.logfile = None args.json = None args.html = None args.cm_file = ih_model analyzed_hits_simple = deepcopy(analyzed_hits) analyzed_hits_locarna = deepcopy(analyzed_hits) analyzed_hits_simple, _, _, _ = extend_simple_core(analyzed_hits_simple, query, blast_args, b_all_short, multi_query, iteration, ih_model) analyzed_hits_locarna, _, _, _ = extend_locarna_core(analyzed_hits_locarna, query, locarna_args, l_all_short, multi_query, iteration, ih_model, timeout=timeout) # add cmstat to query analyzed_hits.query = analyzed_hits_simple.query order_out = [] b_dict = {BA_support.get_hit_n(h): h for h in analyzed_hits_simple.hits} l_dict = {BA_support.get_hit_n(h): h for h in analyzed_hits_locarna.hits} ok_keys = sorted(set(b_dict.keys()) | set(l_dict.keys())) for inum in ok_keys: bh = b_dict.get(inum, None) lh = l_dict.get(inum, None) hits = [bh, lh] # fallback to simple if locarna returned empty hit # deal with the situation when both ways returned empty hits filtered_hits = [h for h in hits if h is not None] if len(filtered_hits) == 1: msg = 'Only one extension method completed successfully for {}. ' \ 'Choosing the successfully extended sequence to the output.'.format(filtered_hits[0].extension.id) ml.info(msg) if ml.getEffectiveLevel() < 20: print(msg) analyzed_hits.hits.append(filtered_hits[0]) continue elif len(filtered_hits) == 0: # append empty extension analyzed_hits.hits_failed.append(lh) continue bit_scores = [ i.extension.annotations['cmstat']['bit_sc'] for i in hits ] mb = max(bit_scores) bit_index = [i for i, j in enumerate(bit_scores) if j == mb][0] order_out.append(bit_index) analyzed_hits.hits.append(hits[bit_index]) # build failed hits b_dict_failed = { BA_support.get_hit_n(h): h for h in analyzed_hits_simple.hits_failed } l_dict_failed = { BA_support.get_hit_n(h): h for h in analyzed_hits_locarna.hits_failed } for inum in sorted(set(b_dict_failed) | set(l_dict_failed)): if inum not in ok_keys: if inum in b_dict_failed: analyzed_hits.hits_failed.append(b_dict_failed[inum]) elif inum in l_dict_failed: analyzed_hits.hits_failed.append(l_dict_failed[inum]) else: raise KeyError( "Failed to find inum key in failed extensions. This should not happen." ) # build the repredict file here if needed if args_inner.repredict_file: b_repredict = BA_support.iter2file_name(blast_args.repredict_file, multi_query, iteration) l_repredict = BA_support.iter2file_name(blast_args.repredict_file, multi_query, iteration) o_repredict = BA_support.iter2file_name(args_inner.repredict_file, multi_query, iteration) with open(b_repredict, 'r') as barf, open(l_repredict, 'r') as larf, open(o_repredict, 'w') as reprf: """ please note that order of files to merge must be same as the order of methods in previous for cycle ie same as the one in which order_out var is set """ bb = (barf, larf) fl = bb[0].readline() reprf.write(fl) fl = bb[0].readline() reprf.write(fl) # dump first line of the other documents [[i.readline() for _ in range(1)] for i in bb[1:]] for o in order_out: lll = [i.readline() for i in bb] reprf.write(lll[o]) # recreate needed data from selected hits homology_prediction = [] homol_seqs = [] for hit in analyzed_hits.hits: homology_prediction.append(hit.hpred) if hit.hpred: homol_seqs.append(hit.extension) # add default prediction if it is not present if 'ss0' not in hit.extension.letter_annotations: if 'sss' not in hit.extension.annotations: hit.extension.anotations['sss'] = [] hit.extension.annotations['sss'] += ['ss0'] hit.extension.letter_annotations['ss0'] = '.' * len( hit.extension.seq) # recreate needed data from selected hits homology_prediction = [] homol_seqs = [] for hit in analyzed_hits.hits: homology_prediction.append(hit.hpred) if hit.hpred: homol_seqs.append(hit.extension) # add default prediction if it is not present if 'ss0' not in hit.extension.letter_annotations: if 'sss' not in hit.extension.annotations: hit.extension.anotations['sss'] = [] hit.extension.annotations['sss'] += ['ss0'] hit.extension.letter_annotations['ss0'] = '.' * len( hit.extension.seq) # remove description from hits and sources for hit in analyzed_hits.hits: hit.extension.description = '' if args_inner.cm_file or args_inner.use_rfam: cm_file_rfam_user = ih_model else: cm_file_rfam_user = None BA_support.remove_one_file_with_try(ih_model) return analyzed_hits, homology_prediction, homol_seqs, cm_file_rfam_user
def extend_simple_core(analyzed_hits, query, args_inner, all_short, multi_query, iteration, ih_model): # the extra here is given "pro forma" the sequence is extended exactly by lenghts of unaligned portions of query if args_inner.db_type == "blastdb": shorts_expanded, _ = rna_blast_analyze.BR_core.extend_hits.expand_hits( all_short, args_inner.blast_db, len(query), extra=0, blast_regexp=args_inner.blast_regexp, skip_missing=args_inner.skip_missing, msgs=analyzed_hits.msgs, ) elif args_inner.db_type in ["fasta", "gb", "server", "entrez"]: shorts_expanded, _ = rna_blast_analyze.BR_core.extend_hits.expand_hits_from_fasta( all_short, args_inner.blast_db, len(query), extra=0, blast_regexp=args_inner.blast_regexp, skip_missing=args_inner.skip_missing, msgs=analyzed_hits.msgs, format=args_inner.db_type, entrez_email=args_inner.entrez, blast_input_file=args_inner.blast_in, ) else: raise exceptions.IncorrectDatabaseChoice() # check, if blast hits are non - overlapping, if so, add the overlapping hit info to the longer hit # reflect this in user output # shorts_expanded = merge_blast_hits(shorts_expanded) shorts_expanded = trim_before(shorts_expanded) shorts_expanded = BA_support.rc_hits_2_rna(shorts_expanded) query_seq = query.seq.transcribe() # blast only extension for exp_hit in shorts_expanded: try: _out = create_blast_only_report_object(exp_hit, len(query_seq)) analyzed_hits.hits.append(_out) except AssertionError as e: exp_hit.annotations['msgs'] += [str(e)] analyzed_hits.hits_failed.append(BA_support.Subsequences(exp_hit)) except exceptions.UnknownStrand as e: exp_hit.annotations['msgs'] += [str(e)] analyzed_hits.hits_failed.append(BA_support.Subsequences(exp_hit)) except Exception as e: ml.error("Unexpected error when extending with 'simple'.") exp_hit.annotations['msgs'] += [str(e)] analyzed_hits.hits_failed.append(BA_support.Subsequences(exp_hit)) if len(analyzed_hits.hits) == 0: ml.error( "Extension failed for all sequences. Please see the error message. You can also try '--mode locarna'." ) sys.exit(1) # assign Locarna score to None as it is not directly accessible from mlocarna for hit in analyzed_hits.hits: hit.extension.annotations['score'] = None # this part predicts homology - it is not truly part of repredict homology_prediction, homol_seqs, cm_file_rfam_user = infer_homology( analyzed_hits=analyzed_hits, args=args_inner, cm_model_file=ih_model, multi_query=multi_query, iteration=iteration) for hit, pred in zip(analyzed_hits.hits, homology_prediction): hit.hpred = pred return analyzed_hits, homology_prediction, homol_seqs, cm_file_rfam_user
def infer_homology(analyzed_hits, args, cm_model_file, multi_query=False, iteration=0): """ This is wrapper for infer homology methods. It deals with different options for generating CM. :return: """ ml.info('Infering homology...') ml.debug(fname()) bits, eval, loc_score, alig_length = hit_cons_characteristic( analyzed_hits.hits) # always run cmscan on rfam for informative reasons # but use inferred CM only if --use_rfam was given # if CM provided, also run inference but use provided file # print explanation alongside this information # find and extract cm model # This code is moved to each extension method to allow fail-fast if model is found in RFAM # cm_model_file, analyzed_hits = find_and_extract_cm_model(args, analyzed_hits) # include query seq in fasta file to get relevant bit score fd_f, fd_fasta = mkstemp(prefix='rba_', suffix='_28', dir=CONFIG.tmpdir) with os.fdopen(fd_f, 'w') as f: for seq in [analyzed_hits.query] + analyzed_hits.res_2_record_list(): f.write('>{}\n{}\n'.format(seq.id, str(seq.seq))) cm_msa, cm_align_scores = run_cmalign_with_scores(fd_fasta, cm_model_file, threads=args.threads) _add_rsearch_align_scores2anal_hits(analyzed_hits, cm_align_scores) # remove first 1 (query) from the prediction scores prediction = infer_hits_cm(cm_align_scores[1:].bit_sc) # write scores to a table, compute it for all data and run some correlation statistics if args.repredict_file: # note that the first score is for the query and act as a benchmark here cm_msa_conservation = alignment_sequence_conservation(cm_msa, gap_chars='-.') repredict_file = BA_support.iter2file_name(args.repredict_file, multi_query, iteration) with open(repredict_file, 'w') as f: _print_table_for_corelation(f, cm_align_scores.seq_name[1:], bits, eval, loc_score, alig_length, cm_msa_conservation[1:], cm_align_scores.bit_sc[1:], cm_msa_conservation[0], cm_align_scores.bit_sc[0]) BA_support.remove_one_file_with_try(fd_fasta) selected_hits = [ hit.extension for b, hit in zip(prediction, analyzed_hits.hits) if b ] if args.cm_file or args.use_rfam: r_cm_file = cm_model_file else: r_cm_file = None BA_support.remove_one_file_with_try(cm_model_file) return prediction, selected_hits, r_cm_file
def create_nr_trusted_hits_file_MSA_safe( sim_threshold_percent=None, all_hits=None, query=None, cmscore_tr=-2.03, cm_threshold_percent=None, check_unambiguous=False, len_diff=0.1, ): """ create non redundant trusted hits file multiple at minimum (2) sequences are needed for profile alignment for some aligners so this function always return two or more sequences or raises exception :param sim_threshold_percent: seq similarity threshold for homology exclusion :param all_hits: list of hits :param query: blast query :param cmscore_tr: threshold for homology inclusion in bits :param cm_threshold_percent: threshold for homology inclusion in % of query bits :param check_unambiguous: bool whether to check unambiguous seqs :param len_diff: threshold for exclusion of hits lq=len(query) lq - diff*lq < len(seq) < lq + diff*lq :return: """ ml.debug(fname()) # i need to leave query, even if with ambiguous basepairs in # because it is used as an reference during distance computation and subsequence selection, # however i don't need to have all homologous seqs there if check_unambiguous: all_hits = BA_support.filter_ambiguous_seqs_from_list(all_hits) dist_table, homologous_seqs, msgs = _trusted_hits_selection_wrapper( all_hits, query, cmscore_tr, cm_threshold_percent, len_diff_=len_diff) if dist_table.size == 0: raise exceptions.NoHomologousSequenceException to_include = rna_blast_analyze.BR_core.predict_structures.select_sequences_from_similarity_rec( dist_table, sim_threshold_percent=sim_threshold_percent) nr_homolog_hits = [homologous_seqs[i] for i in to_include] # final checking of nr homologs # if sequence is filtered here, it is ambiguous basepair in query # removing it is fine if multiple homologous sequences are present # the problem will arise when only 1 homologous sequence will remain # if we added sequence in previous step, raise exception, else behave like in prev step # what if trusted hit is only one? msg = ( 'STATUS: Only one sequence remained under defined "pred_sim_threshold" parameter.\n' ' Mitigation: Adding the most dissimilar homologous sequence to the non redundant sequences list.' ) if len(nr_homolog_hits) < 2 and not check_unambiguous: msgs.append(msg) ml.info(msg) if ml.level > 20: print(msg) dis_hom_index = dist_table[:, 0].argmin() nr_homolog_hits.append( SeqRecord(homologous_seqs[dis_hom_index].seq, id='dummy_seq_01')) del dis_hom_index elif len(nr_homolog_hits) < 2 and check_unambiguous: if len(BA_support.filter_ambiguous_seqs_from_list( nr_homolog_hits)) == 0: # this mean query contain ambiguous bases raise exceptions.NoHomologousSequenceException else: msgs.append(msg) ml.info(msg) if ml.level > 20: print(msg) dis_hom_index = dist_table[:, 0].argmin() nr_homolog_hits.append( SeqRecord(homologous_seqs[dis_hom_index].seq, id='dummy_seq_01')) del dis_hom_index homologous_seqs = BA_support.filter_ambiguous_seqs_from_list( homologous_seqs) elif len(nr_homolog_hits) >= 2 and not check_unambiguous: pass elif len(nr_homolog_hits) > 2 and check_unambiguous: nr_homolog_hits = BA_support.filter_ambiguous_seqs_from_list( nr_homolog_hits) homologous_seqs = BA_support.filter_ambiguous_seqs_from_list( homologous_seqs) elif len(nr_homolog_hits) == 2 and check_unambiguous: homologous_seqs = BA_support.filter_ambiguous_seqs_from_list( homologous_seqs) if len(BA_support.filter_ambiguous_seqs_from_list( nr_homolog_hits)) == 1: # this mean that query contains ambiguous base raise exceptions.NoHomologousSequenceException else: raise Exception() fd_h, nr_homo_hits_file = mkstemp(prefix='rba_', suffix='_58', dir=CONFIG.tmpdir) with os.fdopen(fd_h, 'w') as f: SeqIO.write(nr_homolog_hits, f, 'fasta') return nr_homo_hits_file, homologous_seqs, msgs
def wrapped_ending_with_prediction( args_inner, analyzed_hits, pred_method=None, method_params=None, used_cm_file=None, multi_query=False, iteration=0, ): """ wrapper for prediction of secondary structures :param args_inner: Namespace of input arguments :param analyzed_hits: BlastSearchRecompute object :param pred_method: :param method_params: :param used_cm_file: cmfile if cmfile is known (user given or computed) :return: """ ml.debug(fname()) exec_time = {} msg = 'Entering structure prediction..' if ml.level < 21: ml.info(msg) else: print(msg) ml.info(msg) if pred_method is None: pred_method = args_inner.prediction_method if isinstance(pred_method, str): pred_method = (pred_method, ) if method_params is None: method_params = args_inner.pred_params # ======= filter if needed ======= # do the filtering based on e-val or bitscore # homologous hits still gets used for prediction # annotate ambiguous bases query = BA_support.annotate_ambiguos_base(analyzed_hits.query) # copy the list before filtering all_hits_list = [i.extension for i in analyzed_hits.get_all_hits()] if args_inner.filter_by_eval is not None: hits2predict = filter_by_eval(analyzed_hits.get_all_hits(), BA_support.blast_hit_getter_from_subseq, args_inner.filter_by_eval) _hits = HitList() for h in hits2predict: _hits.append(h) analyzed_hits.hits = _hits elif args_inner.filter_by_bitscore is not None: hits2predict = filter_by_bits(analyzed_hits.get_all_hits(), BA_support.blast_hit_getter_from_subseq, args_inner.filter_by_bitscore) _hits = HitList() for h in hits2predict: _hits.append(h) analyzed_hits.hits = _hits else: analyzed_hits.hits = analyzed_hits.get_all_hits() # if used_cm_file is provided do not override it with CM from RFAM # if use_rfam flag was given, then used_cm_file is already the best_matching model # if analyzed_hits.best_matching_model is None - then we could not find the best matching model in RFAM # and the rfam based methods should fail (i.e. not predict anything) delete_cm = False if used_cm_file is None and analyzed_hits.best_matching_model is not None: rfam = RfamInfo() used_cm_file = run_cmfetch( rfam.file_path, analyzed_hits.best_matching_model['target_name']) delete_cm = True fd, seqs2predict_fasta = mkstemp(prefix='rba_', suffix='_83', dir=CONFIG.tmpdir) with os.fdopen(fd, 'w') as fah: for hit in analyzed_hits.hits: if len(hit.extension.seq) == 0: continue fah.write('>{}\n{}\n'.format(hit.extension.id, str(hit.extension.seq))) if not isinstance(method_params, dict): raise Exception('prediction method parameters must be python dict') # prediction methods present in analyzed_hits # which might be loaded from intermediate file # check if structures of a method are predicted for all required hit # check if the prediction parameters of such method were same prediction_msgs = [] # compute prediction methods which were not computed for pkey in set(pred_method): # add sha1 hashes nh = sha1() nh.update(str(sorted(method_params.get(pkey, {}).items())).encode()) current_hash = nh.hexdigest() if all(pkey in h.extension.letter_annotations for h in analyzed_hits.hits) and \ len( { h.extension.annotations.get('sha1', {}).get(pkey, None) for h in analyzed_hits.hits } | {current_hash, } ) == 1: msg_skip = 'All structures already computed for {}. Skipping...'.format( pkey) ml.info(msg_skip) if ml.level > 20: print(msg_skip, flush=True) continue msg_run = 'Running: {}...'.format(pkey) ml.info(msg_run) if ml.level > 20: print(msg_run, flush=True) structures, etime, msgs = repredict_structures_for_homol_seqs( query, seqs2predict_fasta, args_inner.threads, prediction_method=pkey, pred_method_params=method_params, all_hits_list=all_hits_list, seqs2predict_list=[i.extension for i in analyzed_hits.hits], use_cm_file=used_cm_file, ) exec_time[pkey] = etime if structures is None: msg = 'Structures not predicted with {} method'.format(pkey) ml.info(msg) if ml.level > 20: print('STATUS: ' + msg) else: for i, hit in enumerate(analyzed_hits.hits): assert str(hit.extension.seq) == str(structures[i].seq) hit.extension.annotations['sss'] += [pkey] hit.extension.annotations['msgs'] += structures[ i].annotations.get('msgs', []) # expects "predicted" in annotations - for now, if not given, default is True, as not all prediction # methods implement "predicted" in their output if structures[i].annotations.get('predicted', True): hit.extension.letter_annotations[pkey] = structures[ i].letter_annotations['ss0'] if 'sha1' not in hit.extension.annotations: hit.extension.annotations['sha1'] = dict() hit.extension.annotations['sha1'][pkey] = current_hash try: del hit.extension.letter_annotations['ss0'] except KeyError: pass try: hit.extension.annotations['sss'].remove('ss0') except ValueError: pass analyzed_hits.update_hit_stuctures() # check if msgs are not empty if msgs: prediction_msgs.append('{}: {}'.format(pkey, '\n'.join(msgs))) analyzed_hits.msgs = prediction_msgs with open(args_inner.blast_in + '.r-' + args_inner.sha1[:10], 'r+') as f: all_saved_data = json.load(f) all_saved_data[iteration] = blastsearchrecompute2dict( analyzed_hits) f.seek(0) f.truncate() json.dump(all_saved_data, f, indent=2) # remove structures predicted by different methods (which might be saved from previous computation) for hit in analyzed_hits.hits: for pkey in set(hit.extension.letter_annotations.keys()): if pkey not in pred_method: del hit.extension.letter_annotations[pkey] try: hit.extension.annotations['sss'].remove(pkey) except ValueError: pass BA_support.remove_one_file_with_try(seqs2predict_fasta) if delete_cm: BA_support.remove_one_file_with_try(used_cm_file) add_loc_to_description(analyzed_hits) # write html if requested if args_inner.html: html_file = iter2file_name(args_inner.html, multi_query, iteration) ml.info('Writing html to {}.'.format(html_file)) with open(html_file, 'wb') as h: h.write(write_html_output(analyzed_hits)) # write csv file if requested if args_inner.csv: csv_file = iter2file_name(args_inner.csv, multi_query, iteration) ml.info('Writing csv to {}.'.format(csv_file)) analyzed_hits.to_csv(csv_file) # replace with json if args_inner.json: json_file = iter2file_name(args_inner.json, multi_query, iteration) ml.info('Writing json to {}.'.format(json_file)) j_obj = json.dumps(blastsearchrecompute2dict(analyzed_hits), indent=2) if getattr(args_inner, 'zip_json', False): with open(json_file + '.gz', 'wb') as ff: ff.write(gzip.compress(j_obj.encode())) else: with open(json_file, 'w') as ff: ff.write(j_obj) if args_inner.pandas_dump: pickle_file = iter2file_name(args_inner.pandas_dump, multi_query, iteration) ml.info('Writing pandas pickle to {}.'.format(pickle_file)) pandas.to_pickle(analyzed_hits.pandas, pickle_file) if args_inner.dump: dump_file = iter2file_name(args_inner.dump, multi_query, iteration) ml.info('Writing dump files base: {}.'.format(dump_file)) with open(dump_file, 'wb') as pp: pickle.dump(analyzed_hits, pp, pickle.HIGHEST_PROTOCOL) with open(dump_file + '.time_dump', 'wb') as pp: pickle.dump(exec_time, pp, pickle.HIGHEST_PROTOCOL) return analyzed_hits
def _trusted_hits_selection_wrapper(all_hits_, query_, cmscore_tr_, cm_threshold_percent_, len_diff_=0.1): """ runs basic non_redundant sequences calculation (ie exact sequence match) selects homologous sequences from all hits list by cmscore threshold or by query sequence behaviour: will return distance array with similarities in % including query sequence and list of homologous sequences including query sequence if no sequence is homologous it will return empty array for distance matrix and list with query sequence """ ml.debug(fname()) msgs = [] # trusted sequence selection # ======================================================== assert (cmscore_tr_ == 0) or cm_threshold_percent_ is None score = _extract_cmscore_from_hom_seqs(all_hits_) if cm_threshold_percent_ is not None: selection_threshold = cm_threshold_percent_ * query_.annotations[ 'cmstat'].bit_sc / 100 else: selection_threshold = cmscore_tr_ pred = infer_hits_cm(score, tr=selection_threshold) trusted_seqs_ = [i for i, j in zip(all_hits_, pred) if j] if len(trusted_seqs_) == 0: msg = 'STATUS: No estimated full-length sequences from BLAST output ' \ 'selected as reference for structure prediction.\n' \ ' Using query sequence as reference.' msgs.append(msg) ml.info(msg) if ml.level > 20: print(msg) return np.empty(0), [query_], msgs # add query to trusted sequences trusted_seqs_query = [query_] + trusted_seqs_ # make nr list of sequences -> faster alignment # better selection nr_trusted_seqs_query = BA_support.non_redundant_seqs(trusted_seqs_query) # check if the homologous sequence is not exact match as query # (ie taking non redundant set would be only one sequence) if len(nr_trusted_seqs_query) == 1: msg = 'STATUS: All sequences selected as reference are exactly same as query sequence.' msgs.append(msg) ml.info(msg) if ml.level > 20: print(msg) return np.empty(0), [query_], msgs # select only sequences in some predifined length range to query # this is needed for longish ncRNAs # tolerate 10 % length difference? ref_len = len(query_) nr_len_selected_trusted = [ seq for seq in nr_trusted_seqs_query if ref_len * (1 - len_diff_) < len(seq) < ref_len * (1 + len_diff_) ] # this is to control if only one sequence remained after filtering for length difference if len(nr_len_selected_trusted) == 1: msg = \ 'No sequence satisfy the length difference condition ({}: {}-{})'.format( len_diff_, ref_len * (1 - len_diff_), ref_len * (1 + len_diff_) ) msgs.append(msg) ml.info(msg) if ml.level > 20: print(msg) return np.empty(0), [query_], msgs # sanitize seq names (muscle has issues with too long names) san_hom_seqs, san_dict = BA_support.sanitize_fasta_names_in_seqrec_list( nr_len_selected_trusted) c_fd, trusted_sequence_file_ = mkstemp(prefix='rba_', suffix='_60', dir=CONFIG.tmpdir) with os.fdopen(c_fd, 'w') as f: SeqIO.write(san_hom_seqs, f, 'fasta') align_file = BA_support.run_muscle(trusted_sequence_file_, reorder=True) alig = AlignIO.read(align_file, format='clustal') distance_calc = DistanceCalculator(model='identity') dist_mat = distance_calc.get_distance(alig) # rebuild index from sanitized orig_index = [san_dict[i] for i in dist_mat.names] dist_mat_pd = pandas.DataFrame.from_records(dist_mat.matrix, index=orig_index) dist_table_ = (1 - dist_mat_pd.values) * 100 BA_support.remove_files_with_try([align_file, trusted_sequence_file_]) return dist_table_, trusted_seqs_query, msgs
def extend_locarna_core(analyzed_hits, query, args_inner, all_short, multi_query, iteration, ih_model): # expand hits according to query + 10 nucleotides +- if args_inner.db_type == "blastdb": shorts_expanded, _ = rna_blast_analyze.BR_core.extend_hits.expand_hits( all_short, args_inner.blast_db, len(query), extra=args_inner.subseq_window_locarna, blast_regexp=args_inner.blast_regexp, skip_missing=args_inner.skip_missing, msgs=analyzed_hits.msgs, ) elif args_inner.db_type in ["fasta", "gb", "server", "entrez"]: shorts_expanded, _ = rna_blast_analyze.BR_core.extend_hits.expand_hits_from_fasta( all_short, args_inner.blast_db, len(query), extra=args_inner.subseq_window_locarna, blast_regexp=args_inner.blast_regexp, skip_missing=args_inner.skip_missing, msgs=analyzed_hits.msgs, format=args_inner.db_type, entrez_email=args_inner.entrez, blast_input_file=args_inner.blast_in, ) else: raise exceptions.IncorrectDatabaseChoice() shorts_expanded = BA_support.rc_hits_2_rna(shorts_expanded) query_seq = query.seq.transcribe() # compute alignment here if args_inner.threads == 1: result = [] for oeh in shorts_expanded: result.append( locarna_worker( ( oeh, query_seq, args_inner.locarna_params, args_inner.locarna_anchor_length ) ) ) else: pack = [] for oeh in shorts_expanded: pack.append( ( oeh, query_seq, args_inner.locarna_params, args_inner.locarna_anchor_length ) ) pool = Pool(processes=args_inner.threads) result = pool.map(locarna_worker, pack) pool.close() for res in result: if res.extension is None: analyzed_hits.hits_failed.append(res) else: analyzed_hits.hits.append(res) # this part predicts homology - it is not truly part of repredict homology_prediction, homol_seqs, cm_file_rfam_user = infer_homology( analyzed_hits=analyzed_hits, args=args_inner, cm_model_file=ih_model, multi_query=multi_query, iteration=iteration ) # add homology prediction to the data for hit, pred in zip(analyzed_hits.hits, homology_prediction): hit.hpred = pred return analyzed_hits, homology_prediction, homol_seqs, cm_file_rfam_user
def turbofold_ext_nr_fast(all_seqs, nrset, cpu, n, turbofold_params, pkey='Turbo-fast', sha1val='', timeout=None): # ambiguos sequence cannot be predicted with turbofold # do not predict sequence twice ml.debug(fname()) msg_short_list = 'Turbo-fast: Number of sequences is less then required.' if not len(nrset) == len( BA_support.filter_ambiguous_seqs_from_list(nrset)) == len( BA_support.non_redundant_seqs(nrset)): msgfail = 'Wrong nr set specification.' if len(nrset) != len( BA_support.filter_ambiguous_seqs_from_list(nrset)): msgfail += 'nr set contains seq(s) with ambiguous character.' if len(nrset) != len(BA_support.non_redundant_seqs(nrset)): msgfail += 'nr set contain non redundant sequence(s).' ml.error(msgfail) raise AssertionError(msgfail) for seq in chain(all_seqs, nrset): if 'msgs' not in seq.annotations: seq.annotations['msgs'] = [] list2predict = [] for seq in all_seqs: if seq.annotations.get('ambiguous', False): ml.warning('Skipping prediction for {} (ambiguous base)'.format( seq.id)) continue if pkey in seq.letter_annotations and seq.annotations.get( 'sha1', {}).get(pkey) == sha1val: # nothing to do, the structure already computed continue seq_set = _prepare_set_n(seq, nrset, n) if len(seq_set) < 2: msgfail = "Turbo-fast can't be used with less then 2 sequences - {}".format( seq.id) ml.warning(msgfail) if ml.getEffectiveLevel() > 30: print(msgfail) continue if len(seq_set) < n: msg_short_list_custom = msg_short_list + ' n={} ({})'.format( len(seq_set), n) ml.info(msg_short_list_custom + ' ' + seq.id) seq.annotations['msgs'].append(msg_short_list_custom) list2predict.append((seq_set, turbofold_params, seq.id)) if cpu == 1: pred_list = [] for oneseqset, tpar, _ in list2predict: pred_list.append(run_turbofold(oneseqset, tpar, timeout=timeout)) else: pool = multiprocessing.Pool(processes=cpu) pred_list = pool.map(_rt_wrapper, list2predict) pool.close() # rebuild predicted TurboFold structures # - take care that prediction might be empty if TurboFold fails out_list = [] for out, l_in in zip(pred_list, list2predict): if isinstance(out, exceptions.SubprocessException): seq = next(s for s in all_seqs if s.id == l_in[2]) seq.annotations['msgs'].append(str(out)) # do not propagate call output # seq.annotations['msgs'].append(out.errors) continue elif isinstance(out, Exception): seq = next(s for s in all_seqs if s.id == l_in[2]) seq.annotations['msgs'].append(str(out)) continue sel = [o for o in out if o.id == l_in[2]] if len(sel) == 1: out_list.append(sel[0]) return out_list
def test_nr(self): for seqs_list, check_n in self.test_cases: nr = BA_support.non_redundant_seqs(seqs_list) self.assertEqual(len(nr), len(check_n), 'length do not match') self.assertEqual({str(seq.seq) for seq in nr}, check_n)
def repredict_structures_for_homol_seqs( query, seqs2predict_fasta, threads=None, prediction_method=None, pred_method_params=None, all_hits_list=None, seqs2predict_list=None, use_cm_file=None, ): """Run RNA structure prediction based on chosen method and parameters. """ default_sim_tr_perc = 90 default_score_tr = 0.0 query_max_len_diff = 0.1 try: if 'default' == prediction_method: # do nothing return None, None, [] elif 'rfam-Rc' == prediction_method: if use_cm_file is None: msg = "No CM model. Can't use {}.".format(prediction_method) ml.warning(msg) return None, None, [msg] else: structures, exec_time = cmmodel_rnafold_c( seqs2predict_fasta, use_cm_file, threads=threads, params=pred_method_params.get(prediction_method, {})) return structures, exec_time, [] elif 'rfam-centroid' == prediction_method: # run cmscan if needed # run cmfetch # run cmemit -> homologous seqs # run centroid_homfold method_parameters = pred_method_params.get(prediction_method, {}) if use_cm_file is None: msg = "No CM model. Can't use {}.".format(prediction_method) ml.warning(msg) return None, None, [msg] else: cep = method_parameters.get('cmemit', '') if '-u' not in cep: cep += ' -u' if '-N' not in cep: cep += ' -N {}'.format(method_parameters.get('n_seqs', 10)) hf_file = run_cmemit(use_cm_file, params=cep) structures, exec_time = me_centroid_homfold( seqs2predict_fasta, hf_file, params=method_parameters) BA_support.remove_one_file_with_try(hf_file) return structures, exec_time, [] elif 'rfam-sub' == prediction_method: if use_cm_file is None: msg = "No CM model. Can't use {}.".format(prediction_method) ml.warning(msg) return None, None, [msg] else: ref_structure = extract_ref_from_cm(use_cm_file) structures, exec_time = rfam_subopt_pred( seqs2predict_fasta, ref_structure, params=pred_method_params.get(prediction_method, None), threads=threads, ) return structures, exec_time, [] elif 'rnafold' == prediction_method: structures, exec_time = rnafold_wrap_for_predict( seqs2predict_fasta, params=pred_method_params.get(prediction_method, {}).get('RNAfold', '')) return structures, exec_time, [] elif 'fq-sub' == prediction_method: a, qf = mkstemp(prefix='rba_', suffix='_55', dir=CONFIG.tmpdir) with os.fdopen(a, 'w') as fd: fd.write('>query\n{}\n'.format(str(query.seq))) structures, exec_time = subopt_fold_query( seqs2predict_fasta, qf, params=pred_method_params.get(prediction_method, None), threads=threads) BA_support.remove_one_file_with_try(qf) return structures, exec_time, [] elif 'C-A-sub' == prediction_method: method_parameters = pred_method_params.get(prediction_method, {}) nr_homo_hits_file, homologous_seqs, msgs = create_nr_trusted_hits_file_MSA_safe( all_hits=all_hits_list, query=query, sim_threshold_percent=method_parameters.get( 'pred_sim_threshold', default_sim_tr_perc), cmscore_tr=method_parameters.get('cmscore_tr', default_score_tr), cm_threshold_percent=method_parameters.get( 'cmscore_percent', None), len_diff=method_parameters.get('query_max_len_diff', query_max_len_diff), ) BA_support.remove_one_file_with_try(nr_homo_hits_file) del nr_homo_hits_file f, homologous_sequence_file = mkstemp(prefix='rba_', suffix='_64', dir=CONFIG.tmpdir) with os.fdopen(f, 'w') as fh: SeqIO.write(homologous_seqs, fh, 'fasta') structures, exec_time = subopt_fold_alifold( seqs2predict_fasta, homologous_sequence_file, aligner='clustalo', params=method_parameters, threads=threads) BA_support.remove_one_file_with_try(homologous_sequence_file) del homologous_sequence_file del homologous_seqs return structures, exec_time, msgs elif 'M-A-sub' == prediction_method: method_parameters = pred_method_params.get(prediction_method, {}) nr_homo_hits_file, homologous_seqs, msgs = create_nr_trusted_hits_file_MSA_safe( all_hits=all_hits_list, query=query, sim_threshold_percent=method_parameters.get( 'pred_sim_threshold', default_sim_tr_perc), cmscore_tr=method_parameters.get('cmscore_tr', default_score_tr), cm_threshold_percent=method_parameters.get( 'cmscore_percent', None), len_diff=method_parameters.get('query_max_len_diff', query_max_len_diff), ) BA_support.remove_one_file_with_try(nr_homo_hits_file) del nr_homo_hits_file f, homologous_sequence_file = mkstemp(prefix='rba_', suffix='_65', dir=CONFIG.tmpdir) with os.fdopen(f, 'w') as fh: SeqIO.write(homologous_seqs, fh, 'fasta') structures, exec_time = subopt_fold_alifold( seqs2predict_fasta, homologous_sequence_file, aligner='muscle', params=method_parameters, threads=threads, ) BA_support.remove_one_file_with_try(homologous_sequence_file) del homologous_sequence_file del homologous_seqs return structures, exec_time, msgs elif 'C-A-r-Rc' == prediction_method: method_parameters = pred_method_params.get(prediction_method, {}) nr_homo_hits_file, _, msgs = create_nr_trusted_hits_file_MSA_safe( all_hits=all_hits_list, query=query, sim_threshold_percent=method_parameters.get( 'pred_sim_threshold', default_sim_tr_perc), cmscore_tr=method_parameters.get('cmscore_tr', default_score_tr), cm_threshold_percent=method_parameters.get( 'cmscore_percent', None), len_diff=method_parameters.get('query_max_len_diff', query_max_len_diff), ) structures, exec_time = alifold_refold_prediction( nr_homo_hits_file, seqs2predict_fasta, refold='refold_rnafoldc', threads=threads, params=method_parameters, msa_alg='clustalo') BA_support.remove_one_file_with_try(nr_homo_hits_file) del nr_homo_hits_file return structures, exec_time, msgs elif 'M-A-r-Rc' == prediction_method: method_parameters = pred_method_params.get(prediction_method, {}) nr_homo_hits_file, _, msgs = create_nr_trusted_hits_file_MSA_safe( all_hits=all_hits_list, query=query, sim_threshold_percent=method_parameters.get( 'pred_sim_threshold', default_sim_tr_perc), cmscore_tr=method_parameters.get('cmscore_tr', default_score_tr), cm_threshold_percent=method_parameters.get( 'cmscore_percent', None), len_diff=method_parameters.get('query_max_len_diff', query_max_len_diff), ) structures, exec_time = alifold_refold_prediction( nr_homo_hits_file, seqs2predict_fasta, refold='refold_rnafoldc', threads=threads, params=method_parameters, msa_alg='muscle') BA_support.remove_one_file_with_try(nr_homo_hits_file) del nr_homo_hits_file return structures, exec_time, msgs elif 'C-A-U-r-Rc' == prediction_method: method_parameters = pred_method_params.get(prediction_method, {}) nr_homo_hits_file, _, msgs = create_nr_trusted_hits_file_MSA_safe( all_hits=all_hits_list, query=query, sim_threshold_percent=method_parameters.get( 'pred_sim_threshold', default_sim_tr_perc), cmscore_tr=method_parameters.get('cmscore_tr', default_score_tr), cm_threshold_percent=method_parameters.get( 'cmscore_percent', None), len_diff=method_parameters.get('query_max_len_diff', query_max_len_diff), ) structures, exec_time = alifold_refold_prediction( nr_homo_hits_file, seqs2predict_fasta, refold='conserved_ss_rnafoldc', threads=threads, params=method_parameters, msa_alg='clustalo') BA_support.remove_one_file_with_try(nr_homo_hits_file) del nr_homo_hits_file return structures, exec_time, msgs elif 'M-A-U-r-Rc' == prediction_method: method_parameters = pred_method_params.get(prediction_method, {}) nr_homo_hits_file, _, msgs = create_nr_trusted_hits_file_MSA_safe( all_hits=all_hits_list, query=query, sim_threshold_percent=method_parameters.get( 'pred_sim_threshold', default_sim_tr_perc), cmscore_tr=method_parameters.get('cmscore_tr', default_score_tr), cm_threshold_percent=method_parameters.get( 'cmscore_percent', None), len_diff=method_parameters.get('query_max_len_diff', query_max_len_diff), ) structures, exec_time = alifold_refold_prediction( nr_homo_hits_file, seqs2predict_fasta, refold='conserved_ss_rnafoldc', threads=threads, params=method_parameters, msa_alg='muscle') BA_support.remove_one_file_with_try(nr_homo_hits_file) del nr_homo_hits_file return structures, exec_time, msgs elif 'centroid' == prediction_method: method_parameters = pred_method_params.get(prediction_method, {}) nr_homo_hits_file, _, msgs = create_nr_homolog_hits_file_MSA_unsafe( all_hits=all_hits_list, query=query, sim_threshold_percent=method_parameters.get( 'pred_sim_threshold', default_sim_tr_perc), cmscore_tr=method_parameters.get('cmscore_tr', default_score_tr), cm_threshold_percent=method_parameters.get( 'cmscore_percent', None), len_diff=method_parameters.get('query_max_len_diff', query_max_len_diff), ) raw_structures, exec_time = me_centroid_homfold( seqs2predict_fasta, nr_homo_hits_file, params=method_parameters) # check noncanonical if prediction_method in pred_method_params and pred_method_params[ prediction_method]: allow_nc = pred_method_params[prediction_method].get( 'allow_noncanonical', False) allow_lp = pred_method_params[prediction_method].get( 'allow_lonely_pairs', False) else: allow_nc = False allow_lp = False if not allow_nc: for seq in raw_structures: repstr = find_nc_and_remove( str(seq.seq), structure=seq.letter_annotations['ss0']) seq.letter_annotations['ss0'] = repstr # check lonely basepairs if not allow_lp: for seq in raw_structures: repstr = check_lonely_bp(seq.letter_annotations['ss0']) seq.letter_annotations['ss0'] = repstr BA_support.remove_one_file_with_try(nr_homo_hits_file) del nr_homo_hits_file return raw_structures, exec_time, msgs elif 'centroid-fast' == prediction_method: method_parameters = pred_method_params.get(prediction_method, {}) if query.annotations['ambiguous']: raise exceptions.AmbiguousQuerySequenceException raw_structures, exec_time = centroid_homfold_fast( all_seqs=all_hits_list, query=query, all_seqs_fasta=seqs2predict_fasta, n=method_parameters.get('max_seqs_in_prediction', 10), centroid_homfold_params=method_parameters, len_diff=method_parameters.get('query_max_len_diff', query_max_len_diff)) # check noncanonical if prediction_method in pred_method_params and pred_method_params[ prediction_method]: allow_nc = pred_method_params[prediction_method].get( 'allow_noncanonical', False) allow_lp = pred_method_params[prediction_method].get( 'allow_lonely_pairs', False) else: allow_nc = False allow_lp = False if not allow_nc: for seq in raw_structures: repstr = find_nc_and_remove( str(seq.seq), structure=seq.letter_annotations['ss0']) seq.letter_annotations['ss0'] = repstr # check lonely basepairs if not allow_lp: for seq in raw_structures: repstr = check_lonely_bp(seq.letter_annotations['ss0']) seq.letter_annotations['ss0'] = repstr return raw_structures, exec_time, [] elif 'TurboFold' == prediction_method: # set arbitrary sim_threshold_percent to 100, because we want to remove only identical sequences from prediction # with TurboFold. The structure of redundant sequences will be set according to the one in prediction all_hits_filtered = BA_support.filter_ambiguous_seqs_from_list( all_hits_list) seqs2predict_filtered = BA_support.filter_ambiguous_seqs_from_list( seqs2predict_list) if len(seqs2predict_list) != len(seqs2predict_filtered): ml.warning( 'Some sequences contain ambiguous bases - they will not be predicted.' ) if query.annotations['ambiguous']: raise exceptions.AmbiguousQuerySequenceException() method_parameters = pred_method_params.get(prediction_method, {}) nr_homo_hits_file, _, msgs = create_nr_homolog_hits_file_MSA_unsafe( all_hits=all_hits_filtered, query=query, sim_threshold_percent=100, cmscore_tr=method_parameters.get('cmscore_tr', default_score_tr), cm_threshold_percent=method_parameters.get( 'cmscore_percent', None), len_diff=method_parameters.get('query_max_len_diff', query_max_len_diff), ) with open(nr_homo_hits_file, 'r') as nrf: nr_homo_hits = [ seq for seq in SeqIO.parse(nrf, format='fasta') ] nh = sha1() nh.update(str(sorted(method_parameters.items())).encode()) nh_str = nh.hexdigest() structures_t, exec_time = turbofold_with_homologous( all_sequences=seqs2predict_filtered, nr_homologous=nr_homo_hits, params=method_parameters.get('TurboFold', {}), n=method_parameters.get('max_seqs_in_prediction', 3), cpu=threads, pkey=prediction_method, sha1val=nh_str, ) structures = BA_support.rebuild_structures_output_from_pred( seqs2predict_list, structures_t) BA_support.remove_one_file_with_try(nr_homo_hits_file) del structures_t del nr_homo_hits del nr_homo_hits_file return structures, exec_time, msgs elif 'Turbo-fast' == prediction_method: if query.annotations['ambiguous']: raise exceptions.AmbiguousQuerySequenceException() nh = sha1() nh.update( str( sorted( pred_method_params.get(prediction_method, {}).items())).encode()) nh_str = nh.hexdigest() structures_t, exec_time = turbofold_fast( all_seqs=all_hits_list, seqs2predict=seqs2predict_list, query=query, cpu=threads, n=pred_method_params.get(prediction_method, {}).get('max_seqs_in_prediction', 3), turbofold_params=pred_method_params.get(prediction_method, {}).get( 'TurboFold', {}), len_diff=pred_method_params.get(prediction_method, {}).get( 'query_max_len_diff', query_max_len_diff), pkey=prediction_method, sha1val=nh_str, ) structures = BA_support.rebuild_structures_output_from_pred( seqs2predict_list, structures_t) del structures_t return structures, exec_time, [] except exceptions.NoHomologousSequenceException: msg = nonhomseqwarn(prediction_method) return None, None, [msg] except exceptions.AmbiguousQuerySequenceException: msgfail = "Query sequence contains ambiguous characters. Can't use {}.".format( prediction_method) ml.warning(msgfail) return None, None, [msgfail] except exceptions.SubprocessException as e: msg = "{} can't be used. Error message follows: {} \n{}".format( prediction_method, str(e), e.errors) ml.error(msg) return None, None, [str(e)] except Exception as e: ml.error("{} can't be used. Error message follows: \n{}.".format( prediction_method, str(e))) return None, None, [str(e)] assert False, "Should not reach here (bad prediction method name)."
def locarna_worker(pack): ml.debug(fname()) one_expanded_hit, query_seq, locarna_params, anchor_length = pack locarna_file1 = locarna_file2 = loc_out_file = None try: # read the aligned segment and use it as anchors for locarna # run locarna in local mode and put the query sequence with the extended sequence with the blast aligned # segment as anchor blast_entry = one_expanded_hit.annotations['blast'][1] anchors = LocarnaAnchor( to_rna(blast_entry.query), blast_entry.match, to_rna(blast_entry.sbjct), anchor_length=anchor_length ) if anchors.too_many_anchors: ml.info('Too many anchors for {}. Can handle up to 520 distinct anchors.'.format(one_expanded_hit.id)) # extracted temp is my query # access the locarna aligner directly fd1, locarna_file1 = mkstemp(prefix='rba_', suffix='_20', dir=CONFIG.tmpdir) with os.fdopen(fd1, 'w') as fp_locarna_file_1: ql1, ql2 = anchors.anchor_whole_seq(str(query_seq), 'query') write_clustal_like_file_with_anchors(fp_locarna_file_1, 'query', str(query_seq), ( ('#A1', ql1.split()[0]), ('#A2', ql2.split()[0]) )) fd2, locarna_file2 = mkstemp(prefix='rba_', suffix='_21', dir=CONFIG.tmpdir) with os.fdopen(fd2, 'w') as fp_locarna_file_2: sl1, sl2 = anchors.anchor_whole_seq(str(one_expanded_hit.seq), 'subject') write_clustal_like_file_with_anchors(fp_locarna_file_2, one_expanded_hit.id, str(one_expanded_hit.seq), ( ('#A1', sl1.split()[0]), ('#A2', sl2.split()[0]) )) loc_out_file = run_locarna( locarna_file1, locarna_file2, locarna_params ) # read locarna alignment with open(loc_out_file, 'r') as f: locarna_alig = parse_locarna_alignment(f) if len(locarna_alig) != 2: raise exceptions.SubseqMatchError('There must be 2 sequences in Locarna alignment.') loc_rep = create_report_object_from_locarna(one_expanded_hit, locarna_alig) return loc_rep except exceptions.LocarnaException as e: one_expanded_hit.annotations['msgs'] = [str(e), e.errors] empty_hit = BA_support.Subsequences(one_expanded_hit) return empty_hit except (exceptions.SubseqMatchError, exceptions.ParsingError) as e: one_expanded_hit.annotations['msgs'] = [str(e)] empty_hit = BA_support.Subsequences(one_expanded_hit) return empty_hit except (TypeError, AttributeError, FileNotFoundError) as e: one_expanded_hit.annotations['msgs'] = [str(e)] empty_hit = BA_support.Subsequences(one_expanded_hit) return empty_hit finally: for f in [locarna_file1, locarna_file2, loc_out_file]: if f is not None: BA_support.remove_one_file_with_try(f)
def lunch_computation(args_inner, shared_list=None): ml.debug(fname()) if not shared_list: shared_list = [] # update params if different config is requested CONFIG.override(tools_paths(args_inner.config_file)) p_blast = BA_support.blast_in(args_inner.blast_in, b=args_inner.b_type) query_seqs = [i for i in SeqIO.parse(args_inner.blast_query, 'fasta')] if len(p_blast) != len(query_seqs): ml.error( 'Number of query sequences in provided BLAST output file ({}) does not match number of query sequences' ' in query FASTA file ({}).'.format(len(p_blast), len(query_seqs))) sys.exit(1) # check if BLAST does not contain unexpected sequence characters validate_args.check_blast(p_blast) # create list of correct length if needed all_saved_data = [None] * len(query_seqs) saved_file = '{}.r-{}'.format(args_inner.blast_in, args_inner.sha1[:10]) with open(saved_file, 'r+') as f: _saved = json.load(f) if _saved is None: f.seek(0) f.truncate() json.dump(all_saved_data, f) else: msg = "Loading backup data." print('STATUS: ' + msg) ml.info(msg + ' file: ' + saved_file) all_saved_data = _saved for saved_data in all_saved_data: # we can have partially computed data if saved_data is None: continue if saved_data['args']['sha1'] != args_inner.sha1: msg = "Input argument hash does not match the saved argument hash. " if saved_data['args']['sha1'][:10] == args_inner.sha1[:10]: msg += "This is because of truncating hashes to first 10 characters. " msg += "Please remove the '{}' file.".format( saved_file) ml.error(msg) sys.exit(1) else: msg += "Please remove the '{}' file.".format( saved_file) sys.exit(1) if len(p_blast) > 1: multi_query = True else: multi_query = False # this is done for each query ml_out_line = [] all_analyzed = [] for iteration, (bhp, query, saved_data) in enumerate( zip(p_blast, query_seqs, all_saved_data)): if saved_data is None: print('STATUS: processing query: {}'.format(query.id)) validate_args.verify_query_blast(blast=bhp, query=query) analyzed_hits = BlastSearchRecompute(args_inner, query, iteration) analyzed_hits.multi_query = multi_query # run cm model build # allows to fail fast if rfam was selected and we dont find the model ih_model, analyzed_hits = find_and_extract_cm_model( args_inner, analyzed_hits) # select all all_blast_hits = BA_support.blast_hsps2list(bhp) if len(all_blast_hits) == 0: ml.error('No hits found in {} - {}. Nothing to do.'.format( args_inner.blast_in, bhp.query)) continue # filter if needed if args_inner.filter_by_eval is not None: tmp = filter_by_eval(all_blast_hits, BA_support.blast_hit_getter_from_hits, args_inner.filter_by_eval) if len(tmp) == 0 and len(all_blast_hits) != 0: ml.error( 'The requested filter removed all BLAST hits {} - {}. Nothing to do.' .format(args_inner.blast_in, bhp.query)) continue elif args_inner.filter_by_bitscore is not None: tmp = filter_by_bits(all_blast_hits, BA_support.blast_hit_getter_from_hits, args_inner.filter_by_bitscore) if len(tmp) == 0 and len(all_blast_hits) != 0: ml.error( 'The requested filter removed all BLAST hits {} - {}. Nothing to do.' .format(args_inner.blast_in, bhp.query)) continue all_short = all_blast_hits # now this is different for each mode if args_inner.mode == 'simple': analyzed_hits, homology_prediction, homol_seqs, cm_file_rfam_user = extend_simple_core( analyzed_hits, query, args_inner, all_short, multi_query, iteration, ih_model) elif args_inner.mode == 'locarna': analyzed_hits, homology_prediction, homol_seqs, cm_file_rfam_user = extend_locarna_core( analyzed_hits, query, args_inner, all_short, multi_query, iteration, ih_model) elif args_inner.mode == 'meta': analyzed_hits, homology_prediction, homol_seqs, cm_file_rfam_user = extend_meta_core( analyzed_hits, query, args_inner, all_short, multi_query, iteration, ih_model) else: raise ValueError( 'Unknown option - should be cached by argparse.') if len(analyzed_hits.hits) == 0: ml.error( "Extension failed for all sequences. Please see the error message. You can also try '--mode simple'." ) sys.exit(1) analyzed_hits.copy_hits() with open(args_inner.blast_in + '.r-' + args_inner.sha1[:10], 'r+') as f: all_saved_data = json.load(f) all_saved_data[iteration] = blastsearchrecompute2dict( analyzed_hits) f.seek(0) f.truncate() json.dump(all_saved_data, f, indent=2) else: print( 'STATUS: extended sequences loaded from backup file for query {}' .format(query.id)) analyzed_hits = blastsearchrecomputefromdict(saved_data) # overwrite the saved args with current # this will update used prediction methods and other non essential stuff analyzed_hits.args = args_inner if analyzed_hits.args.cm_file: cm_file_rfam_user = analyzed_hits.args.cm_file else: cm_file_rfam_user = None all_analyzed.append(analyzed_hits) # write all hits to fasta fda, all_hits_fasta = mkstemp(prefix='rba_', suffix='_22', dir=CONFIG.tmpdir) os.close(fda) analyzed_hits.write_results_fasta(all_hits_fasta) out_line = [] # multiple prediction params if args_inner.dev_pred: dp_list = [] # acomodate more dev pred outputs dpfile = None if getattr(args_inner, 'dump', False): dpfile = args_inner.dump.strip('dump') if getattr(args_inner, 'pandas_dump', False): dpfile = args_inner.pandas_dump.strip('pandas_dump') if getattr(args_inner, 'json', False): dpfile = args_inner.json.strip('json') # optimization so the rfam cm file is used only once if cm_file_rfam_user is None and 'rfam' in ''.join( args_inner.prediction_method): best_model = get_cm_model(args_inner.blast_query, threads=args_inner.threads) rfam = RfamInfo() cm_file_rfam_user = run_cmfetch(rfam.file_path, best_model) for method in args_inner.prediction_method: # cycle the prediction method settings # get set of params for each preditcion selected_pred_params = [ kk for kk in args_inner.pred_params if method in kk ] shuffle(selected_pred_params) # for method_params in args_inner.pred_params: for i, method_params in enumerate(selected_pred_params): ah = deepcopy(analyzed_hits) random_flag = BA_support.generate_random_name( 8, shared_list) shared_list.append(random_flag) pname = re.sub(' ', '', str(method)) flag = '|pred_params|' + random_flag # rebuild the args only with actualy used prediction settings ah.args.prediction_method = method ah.args.pred_params = method_params if getattr(args_inner, 'dump', False): spa = args_inner.dump.split('.') ah.args.dump = '.'.join( spa[:-1]) + flag + '.' + spa[-1] if getattr(args_inner, 'pandas_dump', False): spa = args_inner.pandas_dump.split('.') ah.args.pandas_dump = '.'.join( spa[:-1]) + flag + '.' + spa[-1] if getattr(args_inner, 'pdf_out', False): spa = args_inner.pdf_out.split('.') ah.args.pdf_out = '.'.join( spa[:-1]) + flag + '.' + spa[-1] if getattr(args_inner, 'json', False): spa = args_inner.json.split('.') ah.args.json = '.'.join( spa[:-1]) + flag + '.' + spa[-1] wrapped_ending_with_prediction( args_inner=ah.args, analyzed_hits=ah, pred_method=method, method_params=method_params, used_cm_file=cm_file_rfam_user, multi_query=multi_query, iteration=iteration, ) success = True out_line.append(to_tab_delim_line_simple(ah.args)) dp_list.append((i, method_params, success, flag, pname, random_flag, args_inner.pred_params)) if dpfile is not None: with open(dpfile + 'devPredRep', 'wb') as devf: pickle.dump(dp_list, devf) else: wrapped_ending_with_prediction( args_inner=args_inner, analyzed_hits=analyzed_hits, used_cm_file=cm_file_rfam_user, multi_query=multi_query, iteration=iteration, ) out_line.append(to_tab_delim_line_simple(args_inner)) ml_out_line.append('\n'.join(out_line)) if cm_file_rfam_user is not None and os.path.exists(cm_file_rfam_user): BA_support.remove_one_file_with_try(cm_file_rfam_user) BA_support.remove_one_file_with_try(all_hits_fasta) return '\n'.join(ml_out_line), all_analyzed