def __init__(self, handle, __parse_hit_table=False):
self.handle = UndoHandle(handle)
self._preamble = self._parse_preamble()
def __init__(self, filename):
SearchIndexer.__init__(self, filename)
self._handle = UndoHandle(self._handle)
class FastaM10Parser(object):
"""Parser for Bill Pearson's FASTA suite's -m 10 output."""
def __init__(self, handle, __parse_hit_table=False):
self.handle = UndoHandle(handle)
self._preamble = self._parse_preamble()
def __iter__(self):
for qresult in self._parse_qresult():
# re-set desc, for hsp query description
qresult.description = qresult.description
yield qresult
def _parse_preamble(self):
"""Parses the Fasta preamble for Fasta flavor and version."""
preamble = {}
while True:
self.line = self.handle.readline()
# this should be the line just before the first qresult
if self.line.startswith('Query'):
break
# try to match for version line
elif self.line.startswith(' version'):
preamble['version'] = self.line.split(' ')[2]
else:
# try to match for flavor line
flav_match = re.match(_RE_FLAVS, self.line.lower())
if flav_match:
preamble['program'] = flav_match.group(0)
return preamble
def __parse_hit_table(self):
"""Parses hit table rows."""
# move to the first row
self.line = self.handle.readline()
# parse hit table until we see an empty line
hit_rows = []
while self.line and not self.line.strip():
hit_rows.append(self.line.strip())
self.line = self.handle.readline()
return hit_rows
def _parse_qresult(self):
# initial qresult value
qresult = None
hit_rows = []
# state values
state_QRES_NEW = 1
state_QRES_HITTAB = 3
state_QRES_CONTENT = 5
state_QRES_END = 7
while True:
# one line before the hit table
if self.line.startswith('The best scores are:'):
qres_state = state_QRES_HITTAB
# the end of a query or the file altogether
elif self.line.strip() == '>>>///' or not self.line:
qres_state = state_QRES_END
# the beginning of a new query
elif not self.line.startswith('>>>') and '>>>' in self.line:
qres_state = state_QRES_NEW
# the beginning of the query info and its hits + hsps
elif self.line.startswith('>>>') and not \
self.line.strip() == '>>><<<':
qres_state = state_QRES_CONTENT
# default qres mark
else:
qres_state = None
if qres_state is not None:
if qres_state == state_QRES_HITTAB:
# parse hit table if flag is set
hit_rows = self.__parse_hit_table()
elif qres_state == state_QRES_END:
yield _set_qresult_hits(qresult, hit_rows)
break
elif qres_state == state_QRES_NEW:
# if qresult is filled, yield it first
if qresult is not None:
yield _set_qresult_hits(qresult, hit_rows)
regx = re.search(_RE_ID_DESC_SEQLEN, self.line)
query_id = regx.group(1)
seq_len = regx.group(3)
desc = regx.group(2)
qresult = QueryResult(id=query_id)
qresult.seq_len = int(seq_len)
# get target from the next line
self.line = self.handle.readline()
qresult.target = [x for x in self.line.split(' ')
if x][1].strip()
if desc is not None:
qresult.description = desc
# set values from preamble
for key, value in self._preamble.items():
setattr(qresult, key, value)
elif qres_state == state_QRES_CONTENT:
assert self.line[3:].startswith(qresult.id), self.line
for hit, strand in self._parse_hit(query_id):
# HACK: re-set desc, for hsp hit and query description
hit.description = hit.description
hit.query_description = qresult.description
# if hit is not in qresult, append it
if hit.id not in qresult:
qresult.append(hit)
# otherwise, it might be the same hit with a different strand
else:
# make sure strand is different and then append hsp to
# existing hit
for hsp in hit.hsps:
assert strand != hsp.query_strand
qresult[hit.id].append(hsp)
self.line = self.handle.readline()
def _parse_hit(self, query_id):
while True:
self.line = self.handle.readline()
if self.line.startswith('>>'):
break
strand = None
hsp_list = []
while True:
peekline = self.handle.peekline()
# yield hit if we've reached the start of a new query or
# the end of the search
if peekline.strip() in [">>><<<", ">>>///"] or \
(not peekline.startswith('>>>') and '>>>' in peekline):
# append last parsed_hsp['hit']['seq'] line
if state == _STATE_HIT_BLOCK:
parsed_hsp['hit']['seq'] += self.line.strip()
elif state == _STATE_CONS_BLOCK:
hsp.aln_annotation['similarity'] += \
self.line.strip('\r\n')
# process HSP alignment and coordinates
_set_hsp_seqs(hsp, parsed_hsp, self._preamble['program'])
hit = Hit(hsp_list)
hit.description = hit_desc
hit.seq_len = seq_len
yield hit, strand
hsp_list = []
break
# yield hit and create a new one if we're still in the same query
elif self.line.startswith('>>'):
# try yielding, if we have hsps
if hsp_list:
_set_hsp_seqs(hsp, parsed_hsp, self._preamble['program'])
hit = Hit(hsp_list)
hit.description = hit_desc
hit.seq_len = seq_len
yield hit, strand
hsp_list = []
# try to get the hit id and desc, and handle cases without descs
try:
hit_id, hit_desc = self.line[2:].strip().split(' ', 1)
except ValueError:
hit_id = self.line[2:].strip().split(' ', 1)[0]
hit_desc = ''
# create the HSP object for Hit
frag = HSPFragment(hit_id, query_id)
hsp = HSP([frag])
hsp_list.append(hsp)
# set or reset the state to none
state = _STATE_NONE
parsed_hsp = {'query': {}, 'hit': {}}
# create and append a new HSP if line starts with '>--'
elif self.line.startswith('>--'):
# set seq attributes of previous hsp
_set_hsp_seqs(hsp, parsed_hsp, self._preamble['program'])
# and create a new one
frag = HSPFragment(hit_id, query_id)
hsp = HSP([frag])
hsp_list.append(hsp)
# set the state ~ none yet
state = _STATE_NONE
parsed_hsp = {'query': {}, 'hit': {}}
# this is either query or hit data in the HSP, depending on the state
elif self.line.startswith('>'):
if state == _STATE_NONE:
# make sure it's the correct query
assert query_id.startswith(self.line[1:].split(' ')[0]), \
"%r vs %r" % (query_id, self.line)
state = _STATE_QUERY_BLOCK
parsed_hsp['query']['seq'] = ''
elif state == _STATE_QUERY_BLOCK:
# make sure it's the correct hit
assert hit_id.startswith(self.line[1:].split(' ')[0])
state = _STATE_HIT_BLOCK
parsed_hsp['hit']['seq'] = ''
# check for conservation block
elif self.line.startswith('; al_cons'):
state = _STATE_CONS_BLOCK
hsp.fragment.aln_annotation['similarity'] = ''
elif self.line.startswith(';'):
# Fasta outputs do not make a clear distinction between Hit
# and HSPs, so we check the attribute names to determine
# whether it belongs to a Hit or HSP
regx = re.search(_RE_ATTR, self.line.strip())
name = regx.group(1)
value = regx.group(2)
# for values before the '>...' query block
if state == _STATE_NONE:
if name in _HSP_ATTR_MAP:
attr_name, caster = _HSP_ATTR_MAP[name]
if caster is not str:
value = caster(value)
if name in ['_ident', '_sim']:
value *= 100
setattr(hsp, attr_name, value)
# otherwise, pool the values for processing later
elif state == _STATE_QUERY_BLOCK:
parsed_hsp['query'][name] = value
elif state == _STATE_HIT_BLOCK:
if name == '_len':
seq_len = int(value)
else:
parsed_hsp['hit'][name] = value
# for values in the hit block
else:
raise ValueError("Unexpected line: %r" % self.line)
# otherwise, it must be lines containing the sequences
else:
assert '>' not in self.line
# if we're in hit, parse into hsp.hit
if state == _STATE_HIT_BLOCK:
parsed_hsp['hit']['seq'] += self.line.strip()
elif state == _STATE_QUERY_BLOCK:
parsed_hsp['query']['seq'] += self.line.strip()
elif state == _STATE_CONS_BLOCK:
hsp.fragment.aln_annotation['similarity'] += \
self.line.strip('\r\n')
# we should not get here!
else:
raise ValueError("Unexpected line: %r" % self.line)
self.line = self.handle.readline()
def __init__(self, handle, __parse_hit_table=False):
self.handle = UndoHandle(handle)
self._preamble = self._parse_preamble()
class FastaM10Parser(object):
"""Parser for Bill Pearson's FASTA suite's -m 10 output."""
def __init__(self, handle, __parse_hit_table=False):
self.handle = UndoHandle(handle)
self._preamble = self._parse_preamble()
def __iter__(self):
for qresult in self._parse_qresult():
# re-set desc, for hsp query description
qresult.description = qresult.description
yield qresult
def _parse_preamble(self):
"""Parses the Fasta preamble for Fasta flavor and version."""
preamble = {}
while True:
self.line = self.handle.readline()
# this should be the line just before the first qresult
if self.line.startswith('Query'):
break
# try to match for version line
elif self.line.startswith(' version'):
preamble['version'] = self.line.split(' ')[2]
else:
# try to match for flavor line
flav_match = re.match(_RE_FLAVS, self.line.lower())
if flav_match:
preamble['program'] = flav_match.group(0)
return preamble
def __parse_hit_table(self):
"""Parses hit table rows."""
# move to the first row
self.line = self.handle.readline()
# parse hit table until we see an empty line
hit_rows = []
while self.line and not self.line.strip():
hit_rows.append(self.line.strip())
self.line = self.handle.readline()
return hit_rows
def _parse_qresult(self):
# initial qresult value
qresult = None
hit_rows = []
# state values
state_QRES_NEW = 1
state_QRES_HITTAB = 3
state_QRES_CONTENT = 5
state_QRES_END = 7
while True:
# one line before the hit table
if self.line.startswith('The best scores are:'):
qres_state = state_QRES_HITTAB
# the end of a query or the file altogether
elif self.line.strip() == '>>>///' or not self.line:
qres_state = state_QRES_END
# the beginning of a new query
elif not self.line.startswith('>>>') and '>>>' in self.line:
qres_state = state_QRES_NEW
# the beginning of the query info and its hits + hsps
elif self.line.startswith('>>>') and not \
self.line.strip() == '>>><<<':
qres_state = state_QRES_CONTENT
# default qres mark
else:
qres_state = None
if qres_state is not None:
if qres_state == state_QRES_HITTAB:
# parse hit table if flag is set
hit_rows = self.__parse_hit_table()
elif qres_state == state_QRES_END:
yield _set_qresult_hits(qresult, hit_rows)
break
elif qres_state == state_QRES_NEW:
# if qresult is filled, yield it first
if qresult is not None:
yield _set_qresult_hits(qresult, hit_rows)
regx = re.search(_RE_ID_DESC_SEQLEN, self.line)
query_id = regx.group(1)
seq_len = regx.group(3)
desc = regx.group(2)
qresult = QueryResult(id=query_id)
qresult.seq_len = int(seq_len)
# get target from the next line
self.line = self.handle.readline()
qresult.target = [x for x in self.line.split(' ') if x][1].strip()
if desc is not None:
qresult.description = desc
# set values from preamble
for key, value in self._preamble.items():
setattr(qresult, key, value)
elif qres_state == state_QRES_CONTENT:
assert self.line[3:].startswith(qresult.id), self.line
for hit, strand in self._parse_hit(query_id):
# HACK: re-set desc, for hsp hit and query description
hit.description = hit.description
hit.query_description = qresult.description
# if hit is not in qresult, append it
if hit.id not in qresult:
qresult.append(hit)
# otherwise, it might be the same hit with a different strand
else:
# make sure strand is different and then append hsp to
# existing hit
for hsp in hit.hsps:
assert strand != hsp.query_strand
qresult[hit.id].append(hsp)
self.line = self.handle.readline()
def _parse_hit(self, query_id):
while True:
self.line = self.handle.readline()
if self.line.startswith('>>'):
break
strand = None
hsp_list = []
while True:
peekline = self.handle.peekline()
# yield hit if we've reached the start of a new query or
# the end of the search
if peekline.strip() in [">>><<<", ">>>///"] or \
(not peekline.startswith('>>>') and '>>>' in peekline):
# append last parsed_hsp['hit']['seq'] line
if state == _STATE_HIT_BLOCK:
parsed_hsp['hit']['seq'] += self.line.strip()
elif state == _STATE_CONS_BLOCK:
hsp.aln_annotation['similarity'] += \
self.line.strip('\r\n')
# process HSP alignment and coordinates
_set_hsp_seqs(hsp, parsed_hsp, self._preamble['program'])
hit = Hit(hsp_list)
hit.description = hit_desc
hit.seq_len = seq_len
yield hit, strand
hsp_list = []
break
# yield hit and create a new one if we're still in the same query
elif self.line.startswith('>>'):
# try yielding, if we have hsps
if hsp_list:
_set_hsp_seqs(hsp, parsed_hsp, self._preamble['program'])
hit = Hit(hsp_list)
hit.description = hit_desc
hit.seq_len = seq_len
yield hit, strand
hsp_list = []
# try to get the hit id and desc, and handle cases without descs
try:
hit_id, hit_desc = self.line[2:].strip().split(' ', 1)
except ValueError:
hit_id = self.line[2:].strip().split(' ', 1)[0]
hit_desc = ''
# create the HSP object for Hit
frag = HSPFragment(hit_id, query_id)
hsp = HSP([frag])
hsp_list.append(hsp)
# set or reset the state to none
state = _STATE_NONE
parsed_hsp = {'query':{}, 'hit': {}}
# create and append a new HSP if line starts with '>--'
elif self.line.startswith('>--'):
# set seq attributes of previous hsp
_set_hsp_seqs(hsp, parsed_hsp, self._preamble['program'])
# and create a new one
frag = HSPFragment(hit_id, query_id)
hsp = HSP([frag])
hsp_list.append(hsp)
# set the state ~ none yet
state = _STATE_NONE
parsed_hsp = {'query':{}, 'hit': {}}
# this is either query or hit data in the HSP, depending on the state
elif self.line.startswith('>'):
if state == _STATE_NONE:
# make sure it's the correct query
assert query_id.startswith(self.line[1:].split(' ')[0]), \
"%r vs %r" % (query_id, self.line)
state = _STATE_QUERY_BLOCK
parsed_hsp['query']['seq'] = ''
elif state == _STATE_QUERY_BLOCK:
# make sure it's the correct hit
assert hit_id.startswith(self.line[1:].split(' ')[0])
state = _STATE_HIT_BLOCK
parsed_hsp['hit']['seq'] = ''
# check for conservation block
elif self.line.startswith('; al_cons'):
state = _STATE_CONS_BLOCK
hsp.fragment.aln_annotation['similarity'] = ''
elif self.line.startswith(';'):
# Fasta outputs do not make a clear distinction between Hit
# and HSPs, so we check the attribute names to determine
# whether it belongs to a Hit or HSP
regx = re.search(_RE_ATTR, self.line.strip())
name = regx.group(1)
value = regx.group(2)
# for values before the '>...' query block
if state == _STATE_NONE:
if name in _HSP_ATTR_MAP:
attr_name, caster = _HSP_ATTR_MAP[name]
if caster is not str:
value = caster(value)
if name in ['_ident', '_sim']:
value *= 100
setattr(hsp, attr_name, value)
# otherwise, pool the values for processing later
elif state == _STATE_QUERY_BLOCK:
parsed_hsp['query'][name] = value
elif state == _STATE_HIT_BLOCK:
if name == '_len':
seq_len = int(value)
else:
parsed_hsp['hit'][name] = value
# for values in the hit block
else:
raise ValueError("Unexpected line: %r" % self.line)
# otherwise, it must be lines containing the sequences
else:
assert '>' not in self.line
# if we're in hit, parse into hsp.hit
if state == _STATE_HIT_BLOCK:
parsed_hsp['hit']['seq'] += self.line.strip()
elif state == _STATE_QUERY_BLOCK:
parsed_hsp['query']['seq'] += self.line.strip()
elif state == _STATE_CONS_BLOCK:
hsp.fragment.aln_annotation['similarity'] += \
self.line.strip('\r\n')
# we should not get here!
else:
raise ValueError("Unexpected line: %r" % self.line)
self.line = self.handle.readline()
def PdbAtomIterator(handle):
"""Returns SeqRecord objects for each chain in a PDB file
The sequences are derived from the 3D structure (ATOM records), not the
SEQRES lines in the PDB file header.
Unrecognised three letter amino acid codes (e.g. "CSD") from HETATM entries
are converted to "X" in the sequence.
In addition to information from the PDB header (which is the same for all
records), the following chain specific information is placed in the
annotation:
record.annotations["residues"] = List of residue ID strings
record.annotations["chain"] = Chain ID (typically A, B ,...)
record.annotations["model"] = Model ID (typically zero)
Where amino acids are missing from the structure, as indicated by residue
numbering, the sequence is filled in with 'X' characters to match the size
of the missing region, and None is included as the corresponding entry in
the list record.annotations["residues"].
This function uses the Bio.PDB module to do most of the hard work. The
annotation information could be improved but this extra parsing should be
done in parse_pdb_header, not this module.
"""
# Only import PDB when needed, to avoid/delay NumPy dependency in SeqIO
from SAP.Bio.PDB import PDBParser
from SAP.Bio.SeqUtils import seq1
def restype(residue):
"""Return a residue's type as a one-letter code.
Non-standard residues (e.g. CSD, ANP) are returned as 'X'.
"""
return seq1(residue.resname, custom_map=protein_letters_3to1)
# Deduce the PDB ID from the PDB header
# ENH: or filename?
from SAP.Bio.File import UndoHandle
undo_handle = UndoHandle(handle)
firstline = undo_handle.peekline()
if firstline.startswith("HEADER"):
pdb_id = firstline[62:66]
else:
warnings.warn("First line is not a 'HEADER'; can't determine PDB ID")
pdb_id = '????'
struct = PDBParser().get_structure(pdb_id, undo_handle)
model = struct[0]
for chn_id, chain in sorted(model.child_dict.items()):
# HETATM mod. res. policy: remove mod if in sequence, else discard
residues = [
res for res in chain.get_unpacked_list()
if seq1(res.get_resname().upper(), custom_map=protein_letters_3to1)
!= "X"
]
if not residues:
continue
# Identify missing residues in the structure
# (fill the sequence with 'X' residues in these regions)
gaps = []
rnumbers = [r.id[1] for r in residues]
for i, rnum in enumerate(rnumbers[:-1]):
if rnumbers[i + 1] != rnum + 1:
# It's a gap!
gaps.append((i + 1, rnum, rnumbers[i + 1]))
if gaps:
res_out = []
prev_idx = 0
for i, pregap, postgap in gaps:
if postgap > pregap:
gapsize = postgap - pregap - 1
res_out.extend(restype(x) for x in residues[prev_idx:i])
prev_idx = i
res_out.append('X' * gapsize)
else:
warnings.warn("Ignoring out-of-order residues after a gap",
UserWarning)
# Keep the normal part, drop the out-of-order segment
# (presumably modified or hetatm residues, e.g. 3BEG)
res_out.extend(restype(x) for x in residues[prev_idx:i])
break
else:
# Last segment
res_out.extend(restype(x) for x in residues[prev_idx:])
else:
# No gaps
res_out = [restype(x) for x in residues]
record_id = "%s:%s" % (pdb_id, chn_id)
# ENH - model number in SeqRecord id if multiple models?
# id = "Chain%s" % str(chain.id)
# if len(structure) > 1 :
# id = ("Model%s|" % str(model.id)) + id
record = SeqRecord(
Seq(''.join(res_out), generic_protein),
id=record_id,
description=record_id,
)
# The PDB header was loaded as a dictionary, so let's reuse it all
record.annotations = struct.header.copy()
# Plus some chain specifics:
record.annotations["model"] = model.id
record.annotations["chain"] = chain.id
# Start & end
record.annotations["start"] = int(rnumbers[0])
record.annotations["end"] = int(rnumbers[-1])
# ENH - add letter annotations -- per-residue info, e.g. numbers
yield record
def PdbAtomIterator(handle):
"""Returns SeqRecord objects for each chain in a PDB file
The sequences are derived from the 3D structure (ATOM records), not the
SEQRES lines in the PDB file header.
Unrecognised three letter amino acid codes (e.g. "CSD") from HETATM entries
are converted to "X" in the sequence.
In addition to information from the PDB header (which is the same for all
records), the following chain specific information is placed in the
annotation:
record.annotations["residues"] = List of residue ID strings
record.annotations["chain"] = Chain ID (typically A, B ,...)
record.annotations["model"] = Model ID (typically zero)
Where amino acids are missing from the structure, as indicated by residue
numbering, the sequence is filled in with 'X' characters to match the size
of the missing region, and None is included as the corresponding entry in
the list record.annotations["residues"].
This function uses the Bio.PDB module to do most of the hard work. The
annotation information could be improved but this extra parsing should be
done in parse_pdb_header, not this module.
"""
# Only import PDB when needed, to avoid/delay NumPy dependency in SeqIO
from SAP.Bio.PDB import PDBParser
from SAP.Bio.SeqUtils import seq1
def restype(residue):
"""Return a residue's type as a one-letter code.
Non-standard residues (e.g. CSD, ANP) are returned as 'X'.
"""
return seq1(residue.resname, custom_map=protein_letters_3to1)
# Deduce the PDB ID from the PDB header
# ENH: or filename?
from SAP.Bio.File import UndoHandle
undo_handle = UndoHandle(handle)
firstline = undo_handle.peekline()
if firstline.startswith("HEADER"):
pdb_id = firstline[62:66]
else:
warnings.warn("First line is not a 'HEADER'; can't determine PDB ID")
pdb_id = '????'
struct = PDBParser().get_structure(pdb_id, undo_handle)
model = struct[0]
for chn_id, chain in sorted(model.child_dict.items()):
# HETATM mod. res. policy: remove mod if in sequence, else discard
residues = [res for res in chain.get_unpacked_list()
if seq1(res.get_resname().upper(),
custom_map=protein_letters_3to1) != "X"]
if not residues:
continue
# Identify missing residues in the structure
# (fill the sequence with 'X' residues in these regions)
gaps = []
rnumbers = [r.id[1] for r in residues]
for i, rnum in enumerate(rnumbers[:-1]):
if rnumbers[i+1] != rnum + 1:
# It's a gap!
gaps.append((i+1, rnum, rnumbers[i+1]))
if gaps:
res_out = []
prev_idx = 0
for i, pregap, postgap in gaps:
if postgap > pregap:
gapsize = postgap - pregap - 1
res_out.extend(restype(x) for x in residues[prev_idx:i])
prev_idx = i
res_out.append('X'*gapsize)
else:
warnings.warn("Ignoring out-of-order residues after a gap",
UserWarning)
# Keep the normal part, drop the out-of-order segment
# (presumably modified or hetatm residues, e.g. 3BEG)
res_out.extend(restype(x) for x in residues[prev_idx:i])
break
else:
# Last segment
res_out.extend(restype(x) for x in residues[prev_idx:])
else:
# No gaps
res_out = [restype(x) for x in residues]
record_id = "%s:%s" % (pdb_id, chn_id)
# ENH - model number in SeqRecord id if multiple models?
# id = "Chain%s" % str(chain.id)
# if len(structure) > 1 :
# id = ("Model%s|" % str(model.id)) + id
record = SeqRecord(Seq(''.join(res_out), generic_protein),
id=record_id,
description=record_id,
)
# The PDB header was loaded as a dictionary, so let's reuse it all
record.annotations = struct.header.copy()
# Plus some chain specifics:
record.annotations["model"] = model.id
record.annotations["chain"] = chain.id
# Start & end
record.annotations["start"] = int(rnumbers[0])
record.annotations["end"] = int(rnumbers[-1])
# ENH - add letter annotations -- per-residue info, e.g. numbers
yield record
