def get_cm_model_table(query_file,
params=None,
threads=None,
rfam=None,
timeout=None):
ml.debug(fname())
if params is None:
params = dict()
cmscan_params = '-g '
if params and ('cmscan' in params) and params['cmscan']:
cmscan_params += params['cmscan']
try:
out_table = run_cmscan(query_file,
params=cmscan_params,
threads=threads,
rfam=rfam,
timeout=timeout)
f = open(out_table, 'r')
cmscan_data = parse_cmalign_infernal_table(f)
f.close()
remove_one_file_with_try(out_table)
return cmscan_data
except exceptions.CmscanException as e:
return None
def extract_ref_from_cm(cm_file):
ml.debug(fname())
single_alig_file = run_cmemit(cm_file, params='-a -N 1')
o = open(single_alig_file, 'r')
salig = stockholm_read(o)
o.close()
remove_one_file_with_try(single_alig_file)
if len(salig) != 1:
raise AssertionError('File from cmemit does not have only one record in (not including reference).')
# recode structure, return it
ss = salig.column_annotations['SS_cons']
# inserts = str(salig[0].seq)
inserts = str(salig.column_annotations['RF'])
gapchars = '.~'
structure_list = []
for i, j in zip(ss, inserts):
if i in gapchars:
continue
structure_list.append(i)
# recode structure list
structure = cm_strucutre2br(''.join(structure_list))
return structure
def _prepare_pictures(sub):
pictureslist = []
picfile = None
for key in sub.letter_annotations.keys():
np = dict()
np['picname'] = key
np['secondary_structure'] = sub.letter_annotations[key]
try:
picfile = run_rnaplot(seq=str(sub.seq),
structure=sub.letter_annotations[key],
format='svg')
with open(picfile) as f:
np['pic'] = "data:image/svg+xml;utf8," + f.read()
pictureslist.append(np)
remove_one_file_with_try(picfile)
except RNAplotException:
print("can't draw structure with RNAfold for {}.".format(sub.id))
except FileNotFoundError:
if picfile is not None:
print('cannot remove file: {}, file not found'.format(picfile))
except OSError:
if picfile is not None:
print('cannot remove file: {}, file is directory'.format(
picfile))
return pictureslist
def run_rnaplot(seq, structure=None, format='svg', outfile=None):
"""
run rnaplot in desired format
if seq
:param seq:
:param structure:
:return:
"""
ml.debug(fname())
if structure is None:
sequence = str(seq.seq)
structure = seq.letter_annotations['ss0']
else:
sequence = seq
assert len(sequence) == len(structure)
allowed_formats = {'ps', 'svg', 'gml', 'xrna'}
if format not in allowed_formats:
raise TypeError('Format can be only from {}.'.format(allowed_formats))
fd, tmpfile = mkstemp(prefix='rba_', suffix='_08', dir=CONFIG.tmpdir)
rnaname = tmpfile.split('/')[-1].split('\\')[-1]
currdirr = os.getcwd()
tmpdir = gettempdir()
os.chdir(tmpdir)
with os.fdopen(fd, 'w') as fh:
fh.write('>{}\n{}\n{}\n'.format(rnaname, sequence, structure))
cmd = '{} --output-format={} < {}'.format(
shlex.quote('{}RNAplot'.format(CONFIG.viennarna_path)),
shlex.quote(format), shlex.quote(tmpfile))
ml.debug(cmd)
with TemporaryFile(mode='w+', encoding='utf-8') as tmp:
r = call(cmd, shell=True, stdout=tmp, stderr=tmp)
if r:
msgfail = 'Call to RNAplot failed.'
ml.error(msgfail)
os.chdir(currdirr)
tmp.seek(0)
details = tmp.read()
ml.debug(details)
raise exceptions.RNAplotException(msgfail, details)
# output file is name of the sequence (in this case name of the file) + "_ss." + chosen format
remove_one_file_with_try(tmpfile)
plot_output_file = os.path.join(tmpdir, rnaname + '_ss.' + format)
os.chdir(currdirr)
if outfile is None:
return plot_output_file
else:
shutil.move(os.path.join(tmpdir, plot_output_file), outfile)
return outfile
def rnafold_prediction(fasta2predict, params=''):
ml.debug(fname())
fd, structure_output_file = mkstemp(prefix='rba_',
suffix='_54',
dir=CONFIG.tmpdir)
os.close(fd)
structure_output_file = rnafold_fasta(fasta2predict, structure_output_file,
params)
structures = read_seq_str(structure_output_file)
remove_one_file_with_try(structure_output_file)
return structures
def extract_ref_structure_fromRFAM_CM(model_name):
"""
Extract reference structure encoded in covariance model to dot bracket notation.
:param model_name: model name in cm file
:return: string
"""
ml.debug(fname())
rfam = RfamInfo()
single_cm_file = run_cmfetch(rfam.file_path, model_name)
ref_structure = extract_ref_from_cm(single_cm_file)
remove_one_file_with_try(single_cm_file)
return ref_structure
def _run_hybrid_ss_min_wrapper(seq, P, W, M):
fd, tmp_fasta = mkstemp(prefix='rba_', suffix='_06', dir=CONFIG.tmpdir)
try:
with os.fdopen(fd, 'w') as fid:
fid.write('>{}\n{}\n'.format(seq.id, str(seq.seq)))
predicted_ss = _run_hybrid_ss_min_single(tmp_fasta, P, W, M)
return predicted_ss[0]
except exceptions.HybridssminException as e:
return None
finally:
remove_one_file_with_try(tmp_fasta)
def rnafoldc(seqr, constraints_id='cons'):
"""
predict mfe structure with rnafoldc
:param seqr:
:param constraints_id:
:return:
"""
fd, tmpf = mkstemp(prefix='rba_', suffix='_31', dir=CONFIG.tmpdir)
with os.fdopen(fd, 'w') as fh:
fh.write('>seq01\n{}\n{}\n'.format(
str(seqr.seq), seqr.letter_annotations[constraints_id]))
structure = rnafold_prediction(tmpf, params='-C')
remove_one_file_with_try(tmpf)
return structure[0].letter_annotations['ss0']
def run_clustal_profile2seqs_align(msa_file,
fasta_seq_file,
clustalo_params='',
outfile=None):
"""
run clustal align MSA to seqs
aligned columns in input MSA file are preserved and only new sequences are aligned and together they form new
alignment
:param msa_file: msa file (works with stockholm)
:param fasta_seq_file: file with sequences to be aligned (format can be enforced with --infmt in clustalo_params)
:param clustalo_params: params as accepted by clustalo
:param outfile: outfile path, if not provided, tempfile will be created with output
:return: outfile MSA path
"""
ml.info('Runing clustalo profile.')
ml.debug(fname())
def _try_rescue(profile_file):
# beware AlignIO truncates sequence names so they become non-unique, then clustalo also fails
ml.warning(
'Trying rescue for profile alignment if profile has no gaps, sequences appears not aligned. '
'Appending trailing gap to overcome the issue.')
a = AlignIO.read(profile_file, format='clustal')
s = [SeqRecord(Seq(str(i.seq) + '-'), id=i.id) for i in a]
fa = AlignIO.MultipleSeqAlignment(s)
fd, temp = mkstemp(prefix='rba_', suffix='_56', dir=CONFIG.tmpdir)
with os.fdopen(fd, 'w') as fh:
AlignIO.write(fa, fh, format='fasta')
return temp
if outfile:
clustalo_file = outfile
else:
c_fd, clustalo_file = mkstemp(prefix='rba_',
suffix='_57',
dir=CONFIG.tmpdir)
os.close(c_fd)
with TemporaryFile(mode='w+', encoding='utf-8') as tmp:
cmd = [
'{}clustalo'.format(CONFIG.clustal_path), '--force', '-i',
fasta_seq_file, '--profile1', msa_file, '-o', clustalo_file
]
if clustalo_params != '':
cmd += clustalo_params.split()
ml.debug(cmd)
r = call(cmd, stdout=tmp, stderr=tmp)
if r:
ml.warning('Profile align failed.')
# Initiate rescue attempt
rewriten_msa = _try_rescue(msa_file)
cmd2 = [
'{}clustalo'.format(CONFIG.clustal_path), '--force', '-i',
fasta_seq_file, '--profile1', rewriten_msa, '-o', clustalo_file
]
if clustalo_params:
cmd2 += clustalo_params.split()
ml.debug(cmd2)
r2 = call(cmd2, stdout=tmp, stderr=tmp)
remove_one_file_with_try(rewriten_msa)
if r2 != 0:
msgfail = 'Call to clustalo for aligning profile to sequences failed.'
ml.error(msgfail)
ml.error(cmd)
ml.error(cmd2)
raise exceptions.ClustaloException(msgfail, tmp.read())
return clustalo_file
def cmmodel_rnafold_c(allhits_fasta, cmmodel_file, threads=None, params=None):
ml.debug(fname())
if params is None:
params = dict()
allhits_fasta_file, san_dict = sanitize_fasta_file(allhits_fasta)
cmalign_params = ''
if threads:
cmalign_params += '--cpu {}'.format(threads)
if 'cmalign' in params and params['cmalign']:
cmalign_params += ' ' + params['cmalign']
if '--notrunc' not in cmalign_params:
cmalign_params += ' --notrunc'
# rnafold params
rnafold_params = params.get('RNAfold', '-C')
assert isinstance(rnafold_params,
str), "Incorrect parameters for RNAfold -C"
if '-C' not in rnafold_params:
# some parameters given but -C not present
rnafold_params += ' -C'
alig_file = run_cmalign_on_fasta(allhits_fasta_file,
cmmodel_file,
cmalign_params=cmalign_params)
# multiple sequence cm align
# split by sequence, then run the rest
cm_alig = read_st(alig_file)
remove_files_with_try([allhits_fasta_file, alig_file])
structures = []
for single_alig in trim_cmalign_sequence_by_refseq_one_seq(
cm_alig, rs='SS_cons', convert2uppercase=True):
out_alig, trimmed_seq = trim_and_repair_single_cm_alignment(
single_alig)
conserved_structure_pairs = find_nc_and_remove(
str(trimmed_seq.seq), trimmed_seq.letter_annotations['dec_str'])
trimmed_seq.letter_annotations[
'constrains'] = conserved_structure_pairs
# constraint prediction
# write constraint file
fd, temp_constraint_file = mkstemp(prefix='rba_',
suffix='_41',
dir=CONFIG.tmpdir)
with os.fdopen(fd, 'w') as tmpf:
tmpf.write('>{}\n{}\n{}\n'.format(
trimmed_seq.id, str(trimmed_seq.seq),
trimmed_seq.letter_annotations['constrains']))
single_structure = rnafold_prediction(temp_constraint_file,
params=rnafold_params)
remove_one_file_with_try(temp_constraint_file)
# trimmed_seq.letter_annotations['final'] = single_structure[0].letter_annotations['ss0']
structures.append(single_structure[0])
str_out = desanitize_fasta_names_in_seqrec_list(structures, san_dict)
return str_out
def alifold_refold_prediction(nr_homologs_hits_fasta,
all_hits_fasta,
refold='refold',
threads=None,
params=None,
msa_alg='clustalo'):
"""
return predicted structures for all hits based on provided sequence homologs
! beware, clustal mixes order of sequences in profile alignment, correct for it
possible param keys: "clustal", "alifold", "clustalo_profile", "repred_unpaired_tr"
"""
ml.debug(fname())
nr_path, san_dict = sanitize_fasta_file(nr_homologs_hits_fasta)
all_path, san_dict = sanitize_fasta_file(all_hits_fasta,
used_dict=san_dict)
if params is None:
params = dict()
ref_pred = ['refold', 'refold_rnafoldc', 'conserved_ss_rnafoldc']
if refold not in ref_pred:
raise Exception(
'refold procedure not recognized: {}, possible values are {}'.
format(refold, ' '.join(ref_pred)))
cl_file = _aligner_block(nr_path, params, msa_alg, threads)
# cannot rely on that, the order of a cl_file would be the same as the order of the nr_homolog_hits_file
ali_file = compute_alifold(cl_file,
alifold_params=params.get('alifold', ''))
consensus_record = read_seq_str(ali_file)[0]
clustalo_profile_params = '--outfmt clustal '
clustalo_profile_params += params.get('clustalo_profile', '')
if threads:
clustalo_profile_params += ' --threads {}'.format(threads)
realign_file = run_clustal_profile2seqs_align(
cl_file, all_path, clustalo_params=clustalo_profile_params)
realign_alig = AlignIO.read(realign_file, format='clustal')
# slice alignment ( get seqname from nr_homolog_hits_file, find it in the realign and slice the whole segment off
# take care that the id may be the same and it must be checked for multiple occurence
first_nr_record = _parse_first_record_only(nr_path)
realign_allseq_possition = [
i for i, seq in enumerate(realign_alig) if seq.id == first_nr_record.id
]
new_alig_for_refold = realign_alig[:realign_allseq_possition[-1]]
old_alig_in_new = realign_alig[realign_allseq_possition[-1]:]
orig_alignment = AlignIO.read(cl_file, format='clustal')
first_original_alignment_record = orig_alignment[0]
match_original_seq_in_new_alig = [
i for i in old_alig_in_new
if i.id == first_original_alignment_record.id
][0]
mapping = _map_alignment_columns_from_profile_match(
first_original_alignment_record, match_original_seq_in_new_alig)
# map and repair structure when mapping is unbiguous
cs_encode = encode_structure_unicode(
consensus_record.letter_annotations['ss0'])
new_consensus_structure_encoded = _repair_consensus_structure_by_maping(
cs_encode,
mapping,
len(match_original_seq_in_new_alig.seq),
gap_char=49)
new_consensus_structure_repaired = repair_structure_any_variant(
new_consensus_structure_encoded)
new_consensus_structure = decode_structure_unicode(
new_consensus_structure_repaired)
new_consensus_sequence = _repair_consensus_structure_by_maping(
str(consensus_record.seq),
mapping,
len(match_original_seq_in_new_alig.seq),
gap_char=ord('_'))
# write new consensus to a file
a_fd, new_alifold_consensus_file = mkstemp(prefix='rba_',
suffix='_33',
dir=CONFIG.tmpdir)
with os.fdopen(a_fd, 'w') as f:
f.write(new_consensus_sequence + '\n')
f.write(new_consensus_structure + '\n')
# write sliced alignment to a file
sa_fd, sliced_alignment_file = mkstemp(prefix='rba_',
suffix='_34',
dir=CONFIG.tmpdir)
with os.fdopen(sa_fd, 'w') as f:
AlignIO.write(new_alig_for_refold, f, 'clustal')
# now process the file, and map alignment to consensus structure
if refold in ['refold', 'refold_rnafoldc']:
refold_file = compute_refold(sliced_alignment_file,
new_alifold_consensus_file)
if refold == 'refold_rnafoldc':
rnafold_parameters = params.get('RNAfold', '')
if '-C' not in rnafold_parameters:
rnafold_parameters += ' -C'
seq_str = rnafold_prediction(refold_file,
params=rnafold_parameters)
else:
seq_str = read_seq_str(refold_file)
remove_one_file_with_try(refold_file)
else:
st_alig_file = build_stockholm_from_clustal_alig(
sliced_alignment_file, new_alifold_consensus_file)
repred_tr = str(params.get('repred_unpaired_tr', '9'))
conseq_conserved = params.get('conseq_conserved', 1)
seq_str = _refold_with_unpaired_conservation(
st_alig_file,
repred_tr=repred_tr,
conseq_conserved=conseq_conserved)
remove_one_file_with_try(st_alig_file)
structures_out = desanitize_fasta_names_in_seqrec_list(seq_str, san_dict)
remove_files_with_try([
nr_path, all_path, sliced_alignment_file, new_alifold_consensus_file,
cl_file, ali_file, realign_file
])
return structures_out
def expand_hits_from_fasta(hits, database, query_length, extra=0, blast_regexp=None, skip_missing=False, msgs=None, format='fasta', entrez_email=None, blast_input_file=None):
"""takes list of blast.HSP objects and return extended sequences
:return list of SeqRecord objects (parsed fasta file)
"""
ml.info('Retrieving sequence neighborhoods for blast hits.')
ml.debug(fname())
if format == 'server':
# conditional import so we don't need pysam for normal usage
from rna_blast_analyze.BR_core.load_from_bgzip import GenomeDB
seqdb = GenomeDB(database)
if CONFIG.tmpdir is None:
temp_entrez_file = os.path.join(
gettempdir(), os.path.basename(blast_input_file + '.r-temp_entrez')
)
else:
temp_entrez_file = os.path.join(
CONFIG.tmpdir, os.path.basename(blast_input_file + '.r-temp_entrez')
)
if format == 'entrez':
try:
known_seq_index = SeqIO.index(temp_entrez_file, format='fasta')
ml.info("File {} loaded.".format(temp_entrez_file))
if len(known_seq_index) == 0:
remove_one_file_with_try(temp_entrez_file)
except FileNotFoundError:
# ignore that we don't have that file (usual)
known_seq_index = {}
except Exception as e:
ml.info("Could not load the temporary file {}.".format(temp_entrez_file))
known_seq_index = {}
else:
known_seq_index = {}
exp_hits = []
strand = []
for index, hit in enumerate(hits):
# +1 here because blastdbcmd counts sequences from 1
if hit[1].sbjct_end < hit[1].sbjct_start:
# this is hit to minus strand
start = hit[1].sbjct_end - _positive_index(query_length - hit[1].query_end) - extra
end = hit[1].sbjct_start + hit[1].query_start + extra - 1
strand.append(-1)
d = {'query_start': hit[1].sbjct_end, 'query_end': hit[1].sbjct_start,
'extended_start': hit[1].sbjct_end - _positive_index(query_length - hit[1].query_end),
'extended_end': hit[1].sbjct_start + hit[1].query_start - 1,
'strand': -1}
else:
# this is hit to plus strand
start = hit[1].sbjct_start - hit[1].query_start + 1 - extra
end = hit[1].sbjct_end + _positive_index(query_length - hit[1].query_end) + extra
strand.append(1)
d = {'query_start': hit[1].sbjct_start, 'query_end': hit[1].sbjct_end,
'extended_start': hit[1].sbjct_start - hit[1].query_start + 1,
'extended_end': hit[1].sbjct_end + _positive_index(query_length - hit[1].query_end),
'strand': 1}
# ====== information about possible trim ======
# assume ok
d['trimmed_ss'] = False
d['trimmed_se'] = False
d['trimmed_es'] = False
d['trimmed_ee'] = False
d['super_start'] = start
d['super_end'] = end
if start < 1:
start = 1 # index from which sequence should be retrieved from the db
d['trimmed_ss'] = True
# repair possible extended start violation
if d['extended_start'] < 1:
d['trimmed_es'] = True
# add blast record
d['blast'] = hit
try:
bdb_accession = match_acc(hit[0], blast_regexp)
except exceptions.AccessionMatchException as e:
raise e
d['blast'][0] = bdb_accession
# read from file
if format in ['fasta', 'gb']:
ff = os.path.join(database, bdb_accession)
if not os.path.isfile(ff):
if skip_missing:
msgwarn = 'Sequence {} not found in provided db. Skipping.'.format(bdb_accession)
msgs.append(msgwarn)
ml.warning(msgwarn)
else:
msgerror = 'Sequence {} not found in provided db. ' \
'Please provide correct database or give "--skip_missing" flag.'.format(
bdb_accession
)
ml.error(msgerror)
raise LookupError(msgerror)
with open(ff, 'r') as handle:
ext_seq = next(SeqIO.parse(handle, format=format))
parsed_record = ext_seq[start - 1:end]
elif format == 'server':
# only used when server
parsed_record = seqdb.load_genome(bdb_accession, start - 1, end)
elif format == 'entrez':
prnt_line = '{:3d}% {}'.format(floor(index * 100 / len(hits)), bdb_accession)
if index == 0:
sys.stdout.write('STATUS: Downloading required sequences from NCBI with entrez.\n')
sys.stdout.write('{:50}'.format(prnt_line))
else:
sys.stdout.write('\r{:50}'.format(prnt_line))
seq_id = '{}:{}-{}'.format(bdb_accession, start, end)
if seq_id not in known_seq_index:
try:
Entrez.email = entrez_email
parsed_record = fetch_accession_range(bdb_accession, start, end)
with open(temp_entrez_file, 'a') as tmpf:
SeqIO.write([parsed_record], tmpf, format='fasta')
if index == len(hits) - 1:
sys.stdout.write('\r{:50}\n'.format(' Done.'))
except HTTPException as e:
msg = 'HTTP exception encountered: {}' \
'Please check your internet connection and availability of NCBI ENTREZ web services.\n ' \
'Also check that the requested accession number "{}" is available in NCBI "nucleotide" database.'.format(
e, bdb_accession)
if skip_missing:
ml.warning(msg)
continue
else:
ml.error(msg)
sys.exit(1)
except ValueError:
msg = 'Received malformed fasta file. ' \
'Please check that requested accession number "{}" is available in NCBI nucleotide database.'.format(
bdb_accession)
if skip_missing:
ml.warning(msg)
continue
else:
ml.error(msg)
sys.exit(1)
else:
parsed_record = known_seq_index[seq_id]
else:
raise NotImplementedError
record_id = parsed_record.id.split(':')[0]
if parsed_record.description.startswith(parsed_record.id):
parsed_record.description = parsed_record.description[len(parsed_record.id):].strip()
parsed_record.annotations = d
parsed_record.annotations['msgs'] = []
# add uid to ensure that all hits are unique
parsed_record.id = 'uid:' + str(index) + '|' + record_id
if d['trimmed_ss']:
if d['super_start'] + len(parsed_record.seq) < d['super_end'] + d['super_start']:
parsed_record.annotations['trimmed_se'] = True
if d['trimmed_es']:
if len(parsed_record.seq) < d['extended_end'] + d['extended_start']:
parsed_record.annotations['trimmed_ee'] = True
else:
if len(parsed_record.seq) < d['extended_end'] - d['extended_start']:
parsed_record.annotations['trimmed_ee'] = True
else:
if d['super_start'] + len(parsed_record.seq) - 1 < d['super_end']:
parsed_record.annotations['trimmed_se'] = True
if d['super_start'] + len(parsed_record.seq) - 1 < d['extended_end']:
parsed_record.annotations['trimmed_ee'] = True
msgsub = '{}: Sequence cannot be extended sufficiently'.format(parsed_record.id)
if parsed_record.annotations['trimmed_ss']:
msgwarn = msgsub + '. Missing {} nt upstream in the genome.'.format(parsed_record.annotations['super_start'])
parsed_record.annotations['msgs'].append(msgwarn)
ml.warning(msgwarn)
if parsed_record.annotations['trimmed_se']:
msgwarn = msgsub + '. Missing nt downstream in the genome.'.format(parsed_record.id)
parsed_record.annotations['msgs'].append(msgwarn)
ml.warning(msgwarn)
if parsed_record.annotations['trimmed_es']:
msgwarn = msgsub + ' by unaligned portion of query. THIS IS PROBABLY FRAGMENT!'
msgwarn += ' Trimmed upstream.'
parsed_record.annotations['msgs'].append(msgwarn)
ml.warning(msgwarn)
if parsed_record.annotations['trimmed_ee']:
msgwarn = msgsub + ' by unalined portion of query. THIS IS PROBABLY FRAGMENT!'
msgwarn += ' Trimmed downstream.'
parsed_record.annotations['msgs'].append(msgwarn)
ml.warning(msgwarn)
exp_hits.append(parsed_record)
# ==== Remove the entrez tempfile =====
# here we have all the sequences and we can safely delete the tempfile
if format == 'entrez':
remove_one_file_with_try(temp_entrez_file)
return exp_hits, strand