def fetchPDB(pdb, folder='.', compressed=True, copy=False, **kwargs):
"""Retrieve PDB, PDBML, or mmCIF file(s) for specified *pdb* identifier(s).
*pdb* may be a string or a list. The function will return a filename or a
list of filenames depending on input (see :ref:`fetchpdb` for examples).
If *compressed* is ``False``, all files will be decompressed. If *copy* is
``True``, all files from local PDB mirror will copied to the user specified
*folder*. *format* keyword argument can be used to retrieve `PDBML
<http://pdbml.pdb.org/>`_ and `mmCIF <http://mmcif.pdb.org/>`_ files:
``format="cif"`` will fetch an mmCIF file (e.g. :file:`1XXX.cif.gz`),
similarly ``format="xml"`` will fetch a PDBML file. If PDBML header file
is desired, ``format="xml", noatom=True`` will do the job (e.g.
:file:`1XXX-noatom.xml.gz`)
The order of file search operations are as follows: First, files are
sought in *folder*. Second, local PDB mirror will be sought, if one is
set by the user (see :func:`setPDBMirrorPath`). Then, local PDB folder
will be sought, if one is set by the user (see :func:`setPDBLocalFolder`).
Finally, if files are not found locally, they will be downloaded one of
wwPDB FTP servers (use :func:`setWWPDBFTPServer` to specify one close to
you)."""
if isinstance(pdb, str):
identifiers = [pdb]
elif isinstance(pdb, list):
identifiers = pdb
else:
raise TypeError('pdb may be a string or a list of strings')
assert isinstance(folder, str), 'folder must be a string'
assert isinstance(compressed, bool), 'compressed must be a boolean'
assert isinstance(copy, bool), 'copy must be a boolean'
format = kwargs.pop('format', 'pdb')
assert isinstance(format, str), 'format must be a string'
format = format.lower()
assert format in _PDB_FORMATS, '{0:s} is not valid format'.format(
repr(format))
noatom = kwargs.pop('noatom', False)
assert isinstance(noatom, bool), 'noatom must be a boolean'
if kwargs:
raise TypeError('{0:s} is not a valid keyword argument for this'
'function'.format(repr(kwargs.iterkeys().next())))
if folder != '.':
folder = makePath(folder)
if not os.access(folder, os.W_OK):
raise IOError('permission to write in {0:s} is denied, please '
'specify another folder'.format(folder))
filenames = []
exists = 0
success = 0
failure = 0
download = False
if format == 'pdb':
divided = 'data/structures/divided/pdb'
pdbext = '.ent.gz'
extensions = ['.ent', '.pdb'] # '.pdb' should be the last item
prefix = 'pdb'
elif format == 'xml':
if noatom:
divided = 'data/structures/divided/XML-noatom'
pdbext = '-noatom.xml.gz'
extensions = ['-noatom.xml']
else:
divided = 'data/structures/divided/XML'
pdbext = '.xml.gz'
extensions = ['.xml']
prefix = ''
else:
divided = 'data/structures/divided/mmCIF'
pdbext = '.cif.gz'
extensions = ['.cif'] # '.pdb' should be the last item
prefix = ''
pdbfnmap = {}
for extension in extensions:
for pdbfn in glob(os.path.join(folder, '*' + extension + '*')):
if os.path.splitext(pdbfn)[1] in _PDB_EXTENSIONS:
pdbfnmap[os.path.split(pdbfn)[1].split('.')[0].lower()] = pdbfn
for pdbfn in glob(os.path.join(folder, '*' + extension.upper() + '*')):
if os.path.splitext(pdbfn)[1] in _PDB_EXTENSIONS:
pdbfnmap[os.path.split(pdbfn)[1].split('.')[0].lower()] = pdbfn
for i, pdbid in enumerate(identifiers):
# Check validity of identifiers
if not isinstance(pdbid, str):
LOGGER.debug('{0:s} is not a valid identifier.'.format(pdbid))
filenames.append(None)
failure += 1
continue
pdbid = pdbid.strip().lower()
if not (len(pdbid) == 4 and pdbid.isalnum()):
LOGGER.debug('{0:s} is not a valid identifier.'.format(pdbid))
filenames.append(None)
failure += 1
continue
# Check if file exists in working directory
identifiers[i] = pdbid
if noatom:
fn = pdbfnmap.get(pdbid + '-noatom', None)
else:
fn = pdbfnmap.get(pdbid, None) or pdbfnmap.get('pdb'+pdbid, None)
if fn:
fn = relpath(fn)
if not compressed:
temp, ext = os.path.splitext(fn)
if ext == '.gz':
fn = gunzip(fn, temp)
filenames.append(fn)
LOGGER.debug('{0:s} ({1:s}) is found in the working directory.'
.format(pdbid, fn))
exists += 1
continue
# Check the PDB mirror
mirror_path = getPDBMirrorPath()
if mirror_path is not None and os.path.isdir(mirror_path):
fn = os.path.join(mirror_path, divided, pdbid[1:3],
prefix + pdbid + pdbext)
if os.path.isfile(fn):
if copy or not compressed:
if compressed:
filename = os.path.join(folder, pdbid + extension +
'.gz')
shutil.copy(fn, filename)
else:
filename = os.path.join(folder, pdbid + extension)
gunzip(fn, filename)
filenames.append(filename)
LOGGER.debug('{0:s} copied from local mirror ({1:s})'
.format(pdbid, filename))
success += 1
else:
filenames.append(fn)
LOGGER.debug('{0:s} ({1:s}...{2:s}) is found in the local '
'mirror.'.format(pdbid,
fn[:fn[1:].index(os.path.sep)+2], fn[-15:]))
exists += 1
continue
# Check the PDB mirror
local_folder = getPDBLocalFolder()
if format and local_folder:
local_folder, is_divided = local_folder
if is_divided:
fn = os.path.join(local_folder, pdbid[1:3],
'pdb' + pdbid + '.pdb.gz')
else:
fn = os.path.join(local_folder, pdbid + '.pdb.gz')
if os.path.isfile(fn):
if copy or not compressed:
if compressed:
filename = os.path.join(folder, pdbid + extension +
'.gz')
shutil.copy(fn, filename)
else:
filename = os.path.join(folder, pdbid + extension)
gunzip(fn, filename)
filenames.append(filename)
LOGGER.debug('{0:s} copied from local PDB folder ({1:s})'
.format(pdbid, filename))
success += 1
else:
filenames.append(fn)
LOGGER.debug('{0:s} ({1:s}...{2:s}) is found in the PDB '
'local folder.'.format(pdbid,
fn[:fn[1:].index(os.path.sep)+2], fn[-15:]))
exists += 1
continue
filenames.append(pdbid)
download = True
if download:
from ftplib import FTP
ftp_name, ftp_host, ftp_path = getWWPDBFTPServer()
LOGGER.debug('Connecting wwPDB FTP server {0:s}.'.format(ftp_name))
if format == 'pdb' and not copy and local_folder:
folder = local_folder
compressed = True
if is_divided:
getfn = lambda folder, pdbid, ext: \
os.path.join(makePath(os.path.join(local_folder,
pdbid[1:3])), 'pdb' + pdbid + ext)
else:
getfn = lambda folder, pdbid, ext: os.path.join(folder,
pdbid + ext)
else:
getfn = lambda folder, pdbid, ext: os.path.join(folder,
pdbid + ext)
try:
ftp = FTP(ftp_host)
except Exception as error:
raise type(error)('FTP connection problem, potential reason: '
'no internet connectivity')
else:
#ftp_path = os.path.join(ftp_path, divided)
ftp.login('')
for i, pdbid in enumerate(identifiers):
if pdbid != filenames[i]:
continue
filename = getfn(folder, pdbid, extension)
if compressed:
filename += '.gz'
pdbfile = open(filename, 'w+b')
fn = prefix + pdbid + pdbext
try:
ftp.cwd(ftp_path)
ftp.cwd(divided)
ftp.cwd(pdbid[1:3])
ftp.retrbinary('RETR ' + fn, pdbfile.write)
except Exception as error:
pdbfile.close()
os.remove(filename)
if fn in ftp.nlst():
LOGGER.debug('{0:s} download failed ({1:s}). It '
'is possible that you don\'t have '
'rights to download .gz files in the '
'current network.'.format(pdbid,
str(error)))
else:
LOGGER.debug('{0:s} download failed. {1:s} does not '
'exist on {2:s}.'
.format(fn, pdbid, ftp_host))
failure += 1
filenames[i] = None
else:
pdbfile.close()
if not compressed:
gunzip(filename)
filename = relpath(filename)
LOGGER.debug('{0:s} downloaded ({1:s})'
.format(pdbid, filename))
success += 1
filenames[i] = filename
ftp.quit()
if len(identifiers) == 1:
return filenames[0]
else:
LOGGER.info('PDB download completed ({2:d} found, '
'{0:d} downloaded, {1:d} failed).'
.format(success, failure, exists))
return filenames
def fetchPDBLigand(cci, filename=None):
"""Fetch PDB ligand data from PDB_ for chemical component *cci*.
*cci* may be 3-letter chemical component identifier or a valid XML
filename. If *filename* is given, XML file will be saved with that name.
If you query ligand data frequently, you may configure ProDy to save XML
files in your computer. Set ``ligand_xml_save`` option **True**, i.e.
``confProDy(ligand_xml_save=True)``. Compressed XML files will be save
to ProDy package folder, e.g. :file:`/home/user/.prody/pdbligands`. Each
file is around 5Kb when compressed.
This function is compatible with PDBx/PDBML v 4.0.
Ligand data is returned in a dictionary. Ligand coordinate atom data with
*model* and *ideal* coordinate sets are also stored in this dictionary.
Note that this dictionary will contain data that is present in the XML
file and all Ligand Expo XML files do not contain every possible data
field. So, it may be better if you use :meth:`dict.get` instead of
indexing the dictionary, e.g. to retrieve formula weight (or relative
molar mass) of the chemical component use ``data.get('formula_weight')``
instead of ``data['formula_weight']`` to avoid exceptions when this data
field is not found in the XML file. URL and/or path of the XML file are
returned in the dictionary with keys ``url`` and ``path``, respectively.
Following example downloads data for ligand STI (a.k.a. Gleevec and
Imatinib) and calculates RMSD between model (X-ray structure 1IEP) and
ideal (energy minimized) coordinate sets:
.. ipython:: python
from prody import *
ligand_data = fetchPDBLigand('STI')
ligand_data['model_coordinates_db_code']
ligand_model = ligand_data['model']
ligand_ideal = ligand_data['ideal']
transformation = superpose(ligand_ideal.noh, ligand_model.noh)
calcRMSD(ligand_ideal.noh, ligand_model.noh)"""
if not isinstance(cci, str):
raise TypeError('cci must be a string')
if isfile(cci):
inp = openFile(cci)
xml = inp.read()
inp.close()
url = None
path = cci
cci = splitext(splitext(split(cci)[1])[0])[0].upper()
elif len(cci) > 4 or not cci.isalnum():
raise ValueError('cci must be 3-letters long and alphanumeric or '
'a valid filename')
else:
xml = None
cci = cci.upper()
if SETTINGS.get('ligand_xml_save'):
folder = join(getPackagePath(), 'pdbligands')
if not isdir(folder):
makePath(folder)
xmlgz = path = join(folder, cci + '.xml.gz')
if isfile(xmlgz):
with openFile(xmlgz) as inp:
xml = inp.read()
else:
path = None
#url = ('http://ligand-expo.rcsb.org/reports/{0[0]}/{0}/{0}'
# '.xml'.format(cci.upper()))
url = 'http://www.pdb.org/pdb/files/ligand/{0}.xml'.format(cci.upper())
if not xml:
#'http://www.pdb.org/pdb/files/ligand/{0}.xml'
try:
inp = openURL(url)
except IOError:
raise IOError('XML file for ligand {0} is not found online'
.format(cci))
else:
xml = inp.read()
inp.close()
if filename:
out = openFile(filename, mode='w', folder=folder)
out.write(xml)
out.close()
if SETTINGS.get('ligand_xml_save'):
with openFile(xmlgz, 'w') as out:
out.write(xml)
import xml.etree.cElementTree as ET
root = ET.XML(xml)
if (root.get('{http://www.w3.org/2001/XMLSchema-instance}'
'schemaLocation') !=
'http://pdbml.pdb.org/schema/pdbx-v40.xsd pdbx-v40.xsd'):
LOGGER.warn('XML is not in PDBx/PDBML v 4.0 format, resulting '
'dictionary may not contain all data fields')
ns = root.tag[:root.tag.rfind('}')+1]
len_ns = len(ns)
dict_ = {'url': url, 'path': path}
for child in list(root.find(ns + 'chem_compCategory')[0]):
tag = child.tag[len_ns:]
if tag.startswith('pdbx_'):
tag = tag[5:]
dict_[tag] = child.text
dict_['formula_weight'] = float(dict_.get('formula_weight'))
identifiers_and_descriptors = []
results = root.find(ns + 'pdbx_chem_comp_identifierCategory')
if results:
identifiers_and_descriptors.extend(results)
results = root.find(ns + 'pdbx_chem_comp_descriptorCategory')
if results:
identifiers_and_descriptors.extend(results)
for child in identifiers_and_descriptors:
program = child.get('program').replace(' ', '_')
type_ = child.get('type').replace(' ', '_')
dict_[program + '_' + type_] = child[0].text
dict_[program + '_version'] = child.get('program_version')
dict_['audits'] = [(audit.get('action_type'), audit.get('date'))
for audit in
list(root.find(ns + 'pdbx_chem_comp_auditCategory'))]
atoms = list(root.find(ns + 'chem_comp_atomCategory'))
n_atoms = len(atoms)
ideal_coords = np.zeros((n_atoms, 3))
model_coords = np.zeros((n_atoms, 3))
atomnames = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['name'].dtype)
elements = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['element'].dtype)
resnames = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['resname'].dtype)
charges = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['charge'].dtype)
resnums = np.ones(n_atoms, dtype=ATOMIC_FIELDS['charge'].dtype)
alternate_atomnames = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['name'].dtype)
leaving_atom_flags = np.zeros(n_atoms, np.bool)
aromatic_flags = np.zeros(n_atoms, np.bool)
stereo_configs = np.zeros(n_atoms, np.bool)
ordinals = np.zeros(n_atoms, int)
name2index = {}
for i, atom in enumerate(atoms):
data = dict([(child.tag[len_ns:], child.text) for child in list(atom)])
name = data.get('pdbx_component_atom_id', 'X')
name2index[name] = i
atomnames[i] = name
elements[i] = data.get('type_symbol', 'X')
resnames[i] = data.get('pdbx_component_comp_id', 'UNK')
charges[i] = float(data.get('charge', 0))
alternate_atomnames[i] = data.get('alt_atom_id', 'X')
leaving_atom_flags[i] = data.get('pdbx_leaving_atom_flag') == 'Y'
aromatic_flags[i] = data.get('pdbx_atomatic_flag') == 'Y'
stereo_configs[i] = data.get('pdbx_stereo_config') == 'Y'
ordinals[i] = int(data.get('pdbx_ordinal', 0))
model_coords[i, 0] = float(data.get('model_Cartn_x', 0))
model_coords[i, 1] = float(data.get('model_Cartn_y', 0))
model_coords[i, 2] = float(data.get('model_Cartn_z', 0))
ideal_coords[i, 0] = float(data.get('pdbx_model_Cartn_x_ideal', 0))
ideal_coords[i, 1] = float(data.get('pdbx_model_Cartn_y_ideal', 0))
ideal_coords[i, 2] = float(data.get('pdbx_model_Cartn_z_ideal', 0))
pdbid = dict_.get('model_coordinates_db_code')
if pdbid:
model = AtomGroup(cci + ' model ({0})'.format(pdbid))
else:
model = AtomGroup(cci + ' model')
model.setCoords(model_coords)
model.setNames(atomnames)
model.setResnames(resnames)
model.setResnums(resnums)
model.setElements(elements)
model.setCharges(charges)
model.setFlags('leaving_atom_flags', leaving_atom_flags)
model.setFlags('aromatic_flags', aromatic_flags)
model.setFlags('stereo_configs', stereo_configs)
model.setData('ordinals', ordinals)
model.setData('alternate_atomnames', alternate_atomnames)
dict_['model'] = model
ideal = model.copy()
ideal.setTitle(cci + ' ideal')
ideal.setCoords(ideal_coords)
dict_['ideal'] = ideal
bonds = []
warned = set()
for bond in list(root.find(ns + 'chem_comp_bondCategory') or bonds):
name_1 = bond.get('atom_id_1')
name_2 = bond.get('atom_id_2')
try:
bonds.append((name2index[name_1], name2index[name_2]))
except KeyError:
if name_1 not in warned and name_1 not in name2index:
warned.add(name_1)
LOGGER.warn('{0} specified {1} in bond category is not '
'a valid atom name.'.format(repr(name_1), cci))
if name_2 not in warned and name_2 not in name2index:
warned.add(name_2)
LOGGER.warn('{0} specified {1} in bond category is not '
'a valid atom name.'.format(repr(name_2), cci))
if bonds:
bonds = np.array(bonds, int)
model.setBonds(bonds)
ideal.setBonds(bonds)
return dict_
def fetchPfamMSA(acc, alignment="full", compressed=False, **kwargs):
"""Return a path to the downloaded Pfam MSA file.
:arg acc: Pfam ID or Accession Code
:type acc: str
:arg alignment: alignment type, one of ``'full'`` (default), ``'seed'``,
``'ncbi'``, ``'metagenomics'``, ``'rp15'``, ``'rp35'``, ``'rp55'``,
or ``'rp75'`` where rp stands for representative proteomes
:arg compressed: gzip the downloaded MSA file, default is **False**
*Alignment Options*
:arg format: a Pfam supported MSA file format, one of ``'selex'``,
(default), ``'stockholm'`` or ``'fasta'``
:arg order: ordering of sequences, ``'tree'`` (default) or
``'alphabetical'``
:arg inserts: letter case for inserts, ``'upper'`` (default) or ``'lower'``
:arg gaps: gap character, one of ``'dashes'`` (default), ``'dots'``,
``'mixed'`` or **None** for unaligned
*Other Options*
:arg timeout: timeout for blocking connection attempt in seconds, default
is 60
:arg outname: out filename, default is input ``'acc_alignment.format'``
:arg folder: output folder, default is ``'.'``"""
url = "http://pfam.sanger.ac.uk/family/acc?id=" + acc
handle = openURL(url)
orig_acc = acc
acc = handle.readline().strip()
if PY3K:
acc = acc.decode()
url_flag = False
if not re.search("(?<=PF)[0-9]{5}$", acc):
raise ValueError("{0} is not a valid Pfam ID or Accession Code".format(repr(orig_acc)))
if alignment not in DOWNLOAD_FORMATS:
raise ValueError("alignment must be one of full, seed, ncbi or" " metagenomics")
if alignment == "ncbi" or alignment == "metagenomics":
url = "http://pfam.sanger.ac.uk/family/" + acc + "/alignment/" + alignment + "/gzipped"
url_flag = True
extension = ".sth"
else:
if not kwargs:
url = "http://pfam.sanger.ac.uk/family/" + acc + "/alignment/" + alignment + "/gzipped"
url_flag = True
extension = ".sth"
else:
align_format = kwargs.get("format", "selex").lower()
if align_format not in FORMAT_OPTIONS["format"]:
raise ValueError("alignment format must be of type selex" " stockholm or fasta. MSF not supported")
if align_format == SELEX:
align_format, extension = "pfam", ".slx"
elif align_format == FASTA:
extension = ".fasta"
else:
extension = ".sth"
gaps = str(kwargs.get("gaps", "dashes")).lower()
if gaps not in FORMAT_OPTIONS["gaps"]:
raise ValueError("gaps must be of type mixed, dots, dashes, " "or None")
inserts = kwargs.get("inserts", "upper").lower()
if inserts not in FORMAT_OPTIONS["inserts"]:
raise ValueError("inserts must be of type lower or upper")
order = kwargs.get("order", "tree").lower()
if order not in FORMAT_OPTIONS["order"]:
raise ValueError("order must be of type tree or alphabetical")
url = (
"http://pfam.sanger.ac.uk/family/"
+ acc
+ "/alignment/"
+ alignment
+ "/format?format="
+ align_format
+ "&alnType="
+ alignment
+ "&order="
+ order[0]
+ "&case="
+ inserts[0]
+ "&gaps="
+ gaps
+ "&download=1"
)
response = openURL(url, timeout=int(kwargs.get("timeout", 60)))
outname = kwargs.get("outname", None)
if not outname:
outname = orig_acc
folder = str(kwargs.get("folder", "."))
filepath = join(makePath(folder), outname + "_" + alignment + extension)
if compressed:
filepath = filepath + ".gz"
if url_flag:
f_out = open(filepath, "wb")
else:
f_out = openFile(filepath, "wb")
f_out.write(response.read())
f_out.close()
else:
if url_flag:
gunzip(response.read(), filepath)
else:
with open(filepath, "wb") as f_out:
f_out.write(response.read())
filepath = relpath(filepath)
LOGGER.info("Pfam MSA for {0} is written as {1}.".format(orig_acc, filepath))
return filepath
def fetchPfamMSA(acc, alignment='full', compressed=False, **kwargs):
"""Return a path to the downloaded Pfam MSA file.
:arg acc: Pfam ID or Accession Code
:type acc: str
:arg alignment: alignment type, one of ``'full'`` (default), ``'seed'``,
``'ncbi'``, ``'metagenomics'``, ``'rp15'``, ``'rp35'``, ``'rp55'``,
or ``'rp75'`` where rp stands for representative proteomes
:arg compressed: gzip the downloaded MSA file, default is **False**
*Alignment Options*
:arg format: a Pfam supported MSA file format, one of ``'selex'``,
(default), ``'stockholm'`` or ``'fasta'``
:arg order: ordering of sequences, ``'tree'`` (default) or
``'alphabetical'``
:arg inserts: letter case for inserts, ``'upper'`` (default) or ``'lower'``
:arg gaps: gap character, one of ``'dashes'`` (default), ``'dots'``,
``'mixed'`` or **None** for unaligned
*Other Options*
:arg timeout: timeout for blocking connection attempt in seconds, default
is 60
:arg outname: out filename, default is input ``'acc_alignment.format'``
:arg folder: output folder, default is ``'.'``"""
url = 'http://pfam.sanger.ac.uk/family/acc?id=' + acc
handle = openURL(url)
orig_acc = acc
acc = handle.readline().strip()
if PY3K:
acc = acc.decode()
url_flag = False
if not re.search('(?<=PF)[0-9]{5}$', acc):
raise ValueError('{0} is not a valid Pfam ID or Accession Code'
.format(repr(orig_acc)))
if alignment not in DOWNLOAD_FORMATS:
raise ValueError('alignment must be one of full, seed, ncbi or'
' metagenomics')
if alignment == 'ncbi' or alignment == 'metagenomics':
url = ('http://pfam.sanger.ac.uk/family/' + acc + '/alignment/' +
alignment + '/gzipped')
url_flag = True
extension = '.sth'
else:
if not kwargs:
url = ('http://pfam.sanger.ac.uk/family/' + acc + '/alignment/' +
alignment + '/gzipped')
url_flag = True
extension = '.sth'
else:
align_format = kwargs.get('format', 'selex').lower()
if align_format not in FORMAT_OPTIONS['format']:
raise ValueError('alignment format must be of type selex'
' stockholm or fasta. MSF not supported')
if align_format == SELEX:
align_format, extension = 'pfam', '.slx'
elif align_format == FASTA:
extension = '.fasta'
else:
extension = '.sth'
gaps = str(kwargs.get('gaps', 'dashes')).lower()
if gaps not in FORMAT_OPTIONS['gaps']:
raise ValueError('gaps must be of type mixed, dots, dashes, '
'or None')
inserts = kwargs.get('inserts', 'upper').lower()
if(inserts not in FORMAT_OPTIONS['inserts']):
raise ValueError('inserts must be of type lower or upper')
order = kwargs.get('order', 'tree').lower()
if order not in FORMAT_OPTIONS['order']:
raise ValueError('order must be of type tree or alphabetical')
url = ('http://pfam.sanger.ac.uk/family/' + acc + '/alignment/'
+ alignment + '/format?format=' + align_format +
'&alnType=' + alignment + '&order=' + order[0] +
'&case=' + inserts[0] + '&gaps=' + gaps + '&download=1')
response = openURL(url, timeout=int(kwargs.get('timeout', 60)))
outname = kwargs.get('outname', None)
if not outname:
outname = orig_acc
folder = str(kwargs.get('folder', '.'))
filepath = join(makePath(folder), outname + '_' + alignment + extension)
if compressed:
filepath = filepath + '.gz'
if url_flag:
f_out = open(filepath, 'wb')
else:
f_out = openFile(filepath, 'wb')
f_out.write(response.read())
f_out.close()
else:
if url_flag:
gunzip(response.read(), filepath)
else:
with open(filepath, 'wb') as f_out:
f_out.write(response.read())
filepath = relpath(filepath)
LOGGER.info('Pfam MSA for {0} is written as {1}.'
.format(orig_acc, filepath))
return filepath
def fetchPfamMSA(acc, alignment='full', compressed=False, **kwargs):
"""Return a path to the downloaded Pfam MSA file.
:arg acc: Pfam ID or Accession Code
:type acc: str
:arg alignment: alignment type, one of ``'full'`` (default), ``'seed'``,
``'ncbi'``, ``'metagenomics'``, ``'rp15'``, ``'rp35'``, ``'rp55'``,
or ``'rp75'`` where rp stands for representative proteomes
:arg compressed: gzip the downloaded MSA file, default is **False**
*Alignment Options*
:arg format: a Pfam supported MSA file format, one of ``'selex'``,
(default), ``'stockholm'`` or ``'fasta'``
:arg order: ordering of sequences, ``'tree'`` (default) or
``'alphabetical'``
:arg inserts: letter case for inserts, ``'upper'`` (default) or ``'lower'``
:arg gaps: gap character, one of ``'dashes'`` (default), ``'dots'``,
``'mixed'`` or **None** for unaligned
*Other Options*
:arg timeout: timeout for blocking connection attempt in seconds, default
is 60
:arg outname: out filename, default is input ``'acc_alignment.format'``
:arg folder: output folder, default is ``'.'``"""
url = 'http://pfam.sanger.ac.uk/family/acc?id=' + acc
handle = openURL(url)
orig_acc = acc
acc = handle.readline().strip()
if PY3K:
acc = acc.decode()
url_flag = False
if not re.search('(?<=PF)[0-9]{5}$', acc):
raise ValueError('{0} is not a valid Pfam ID or Accession Code'
.format(repr(orig_acc)))
if alignment not in DOWNLOAD_FORMATS:
raise ValueError('alignment must be one of full, seed, ncbi or'
' metagenomics')
if alignment == 'ncbi' or alignment == 'metagenomics':
url = ('http://pfam.sanger.ac.uk/family/' + acc + '/alignment/' +
alignment + '/gzipped')
url_flag = True
extension = '.sth'
else:
if not kwargs:
url = ('http://pfam.sanger.ac.uk/family/' + acc + '/alignment/' +
alignment + '/gzipped')
url_flag = True
extension = '.sth'
else:
align_format = kwargs.get('format', 'selex').lower()
if align_format not in FORMAT_OPTIONS['format']:
raise ValueError('alignment format must be of type selex'
' stockholm or fasta. MSF not supported')
if align_format == SELEX:
align_format, extension = 'pfam', '.slx'
elif align_format == FASTA:
extension = '.fasta'
else:
extension = '.sth'
gaps = str(kwargs.get('gaps', 'dashes')).lower()
if gaps not in FORMAT_OPTIONS['gaps']:
raise ValueError('gaps must be of type mixed, dots, dashes, '
'or None')
inserts = kwargs.get('inserts', 'upper').lower()
if(inserts not in FORMAT_OPTIONS['inserts']):
raise ValueError('inserts must be of type lower or upper')
order = kwargs.get('order', 'tree').lower()
if order not in FORMAT_OPTIONS['order']:
raise ValueError('order must be of type tree or alphabetical')
url = ('http://pfam.sanger.ac.uk/family/' + acc + '/alignment/'
+ alignment + '/format?format=' + align_format +
'&alnType=' + alignment + '&order=' + order[0] +
'&case=' + inserts[0] + '&gaps=' + gaps + '&download=1')
response = openURL(url, timeout=int(kwargs.get('timeout', 60)))
outname = kwargs.get('outname', None)
if not outname:
outname = orig_acc
folder = str(kwargs.get('folder', '.'))
filepath = join(makePath(folder), outname + '_' + alignment + extension)
if compressed:
filepath = filepath + '.gz'
if url_flag:
f_out = open(filepath, 'wb')
else:
f_out = openFile(filepath, 'wb')
f_out.write(response.read())
f_out.close()
else:
if url_flag:
gunzip(response.read(), filepath)
else:
with open(filepath, 'wb') as f_out:
f_out.write(response.read())
filepath = relpath(filepath)
LOGGER.info('Pfam MSA for {0} is written as {1}.'
.format(orig_acc, filepath))
return filepath
def fetchPDBLigand(cci, filename=None):
"""Fetch PDB ligand data from PDB_ for chemical component *cci*.
*cci* may be 3-letter chemical component identifier or a valid XML
filename. If *filename* is given, XML file will be saved with that name.
If you query ligand data frequently, you may configure ProDy to save XML
files in your computer. Set ``ligand_xml_save`` option **True**, i.e.
``confProDy(ligand_xml_save=True)``. Compressed XML files will be save
to ProDy package folder, e.g. :file:`/home/user/.prody/pdbligands`. Each
file is around 5Kb when compressed.
This function is compatible with PDBx/PDBML v 4.0.
Ligand data is returned in a dictionary. Ligand coordinate atom data with
*model* and *ideal* coordinate sets are also stored in this dictionary.
Note that this dictionary will contain data that is present in the XML
file and all Ligand Expo XML files do not contain every possible data
field. So, it may be better if you use :meth:`dict.get` instead of
indexing the dictionary, e.g. to retrieve formula weight (or relative
molar mass) of the chemical component use ``data.get('formula_weight')``
instead of ``data['formula_weight']`` to avoid exceptions when this data
field is not found in the XML file. URL and/or path of the XML file are
returned in the dictionary with keys ``url`` and ``path``, respectively.
Following example downloads data for ligand STI (a.k.a. Gleevec and
Imatinib) and calculates RMSD between model (X-ray structure 1IEP) and
ideal (energy minimized) coordinate sets:
.. ipython:: python
from prody import *
ligand_data = fetchPDBLigand('STI')
ligand_data['model_coordinates_db_code']
ligand_model = ligand_data['model']
ligand_ideal = ligand_data['ideal']
transformation = superpose(ligand_ideal.noh, ligand_model.noh)
calcRMSD(ligand_ideal.noh, ligand_model.noh)"""
if not isinstance(cci, str):
raise TypeError('cci must be a string')
if isfile(cci):
inp = openFile(cci)
xml = inp.read()
inp.close()
url = None
path = cci
cci = splitext(splitext(split(cci)[1])[0])[0].upper()
elif len(cci) > 4 or not cci.isalnum():
raise ValueError('cci must be 3-letters long and alphanumeric or '
'a valid filename')
else:
xml = None
cci = cci.upper()
if SETTINGS.get('ligand_xml_save'):
folder = join(getPackagePath(), 'pdbligands')
if not isdir(folder):
makePath(folder)
xmlgz = path = join(folder, cci + '.xml.gz')
if isfile(xmlgz):
with openFile(xmlgz) as inp:
xml = inp.read()
else:
path = None
#url = ('http://ligand-expo.rcsb.org/reports/{0[0]}/{0}/{0}'
# '.xml'.format(cci.upper()))
url = 'http://files.rcsb.org/ligands/download/{0}.xml'.format(
cci.upper())
if not xml:
#'http://www.pdb.org/pdb/files/ligand/{0}.xml'
try:
inp = openURL(url)
except IOError:
raise IOError(
'XML file for ligand {0} is not found online'.format(cci))
else:
xml = inp.read()
inp.close()
if filename:
out = openFile(filename, mode='w', folder=folder)
out.write(xml)
out.close()
if SETTINGS.get('ligand_xml_save'):
with openFile(xmlgz, 'w') as out:
out.write(xml)
import xml.etree.cElementTree as ET
root = ET.XML(xml)
if (root.get('{http://www.w3.org/2001/XMLSchema-instance}'
'schemaLocation') !=
'http://pdbml.pdb.org/schema/pdbx-v40.xsd pdbx-v40.xsd'):
LOGGER.warn('XML is not in PDBx/PDBML v 4.0 format, resulting '
'dictionary may not contain all data fields')
ns = root.tag[:root.tag.rfind('}') + 1]
len_ns = len(ns)
dict_ = {'url': url, 'path': path}
for child in list(root.find(ns + 'chem_compCategory')[0]):
tag = child.tag[len_ns:]
if tag.startswith('pdbx_'):
tag = tag[5:]
dict_[tag] = child.text
dict_['formula_weight'] = float(dict_.get('formula_weight'))
identifiers_and_descriptors = []
results = root.find(ns + 'pdbx_chem_comp_identifierCategory')
if results:
identifiers_and_descriptors.extend(results)
results = root.find(ns + 'pdbx_chem_comp_descriptorCategory')
if results:
identifiers_and_descriptors.extend(results)
for child in identifiers_and_descriptors:
program = child.get('program').replace(' ', '_')
type_ = child.get('type').replace(' ', '_')
dict_[program + '_' + type_] = child[0].text
dict_[program + '_version'] = child.get('program_version')
dict_['audits'] = [
(audit.get('action_type'), audit.get('date'))
for audit in list(root.find(ns + 'pdbx_chem_comp_auditCategory'))
]
atoms = list(root.find(ns + 'chem_comp_atomCategory'))
n_atoms = len(atoms)
ideal_coords = np.zeros((n_atoms, 3))
model_coords = np.zeros((n_atoms, 3))
atomnames = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['name'].dtype)
elements = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['element'].dtype)
resnames = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['resname'].dtype)
charges = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['charge'].dtype)
resnums = np.ones(n_atoms, dtype=ATOMIC_FIELDS['charge'].dtype)
alternate_atomnames = np.zeros(n_atoms, dtype=ATOMIC_FIELDS['name'].dtype)
leaving_atom_flags = np.zeros(n_atoms, np.bool)
aromatic_flags = np.zeros(n_atoms, np.bool)
stereo_configs = np.zeros(n_atoms, np.bool)
ordinals = np.zeros(n_atoms, int)
name2index = {}
for i, atom in enumerate(atoms):
data = dict([(child.tag[len_ns:], child.text) for child in list(atom)])
name = data.get('pdbx_component_atom_id', 'X')
name2index[name] = i
atomnames[i] = name
elements[i] = data.get('type_symbol', 'X')
resnames[i] = data.get('pdbx_component_comp_id', 'UNK')
charges[i] = float(data.get('charge', 0))
alternate_atomnames[i] = data.get('alt_atom_id', 'X')
leaving_atom_flags[i] = data.get('pdbx_leaving_atom_flag') == 'Y'
aromatic_flags[i] = data.get('pdbx_atomatic_flag') == 'Y'
stereo_configs[i] = data.get('pdbx_stereo_config') == 'Y'
ordinals[i] = int(data.get('pdbx_ordinal', 0))
model_coords[i, 0] = float(data.get('model_Cartn_x', 0))
model_coords[i, 1] = float(data.get('model_Cartn_y', 0))
model_coords[i, 2] = float(data.get('model_Cartn_z', 0))
ideal_coords[i, 0] = float(data.get('pdbx_model_Cartn_x_ideal', 0))
ideal_coords[i, 1] = float(data.get('pdbx_model_Cartn_y_ideal', 0))
ideal_coords[i, 2] = float(data.get('pdbx_model_Cartn_z_ideal', 0))
pdbid = dict_.get('model_coordinates_db_code')
if pdbid:
model = AtomGroup(cci + ' model ({0})'.format(pdbid))
else:
model = AtomGroup(cci + ' model')
model.setCoords(model_coords)
model.setNames(atomnames)
model.setResnames(resnames)
model.setResnums(resnums)
model.setElements(elements)
model.setCharges(charges)
model.setFlags('leaving_atom_flags', leaving_atom_flags)
model.setFlags('aromatic_flags', aromatic_flags)
model.setFlags('stereo_configs', stereo_configs)
model.setData('ordinals', ordinals)
model.setData('alternate_atomnames', alternate_atomnames)
dict_['model'] = model
ideal = model.copy()
ideal.setTitle(cci + ' ideal')
ideal.setCoords(ideal_coords)
dict_['ideal'] = ideal
bonds = []
warned = set()
for bond in list(root.find(ns + 'chem_comp_bondCategory') or bonds):
name_1 = bond.get('atom_id_1')
name_2 = bond.get('atom_id_2')
try:
bonds.append((name2index[name_1], name2index[name_2]))
except KeyError:
if name_1 not in warned and name_1 not in name2index:
warned.add(name_1)
LOGGER.warn('{0} specified {1} in bond category is not '
'a valid atom name.'.format(repr(name_1), cci))
if name_2 not in warned and name_2 not in name2index:
warned.add(name_2)
LOGGER.warn('{0} specified {1} in bond category is not '
'a valid atom name.'.format(repr(name_2), cci))
if bonds:
bonds = np.array(bonds, int)
model.setBonds(bonds)
ideal.setBonds(bonds)
return dict_
def fetchPDBviaHTTP(*pdb, **kwargs):
"""Retrieve PDB file(s) for specified *pdb* identifier(s) and return
path(s). Downloaded files will be stored in local PDB folder, if one
is set using :meth:`.pathPDBFolder`, and copied into *folder*, if
specified by the user. If no destination folder is specified, files
will be saved in the current working directory. If *compressed* is
**False**, decompressed files will be copied into *folder*."""
if kwargs.get('check', True):
identifiers = checkIdentifiers(*pdb)
else:
identifiers = list(pdb)
output_folder = kwargs.pop('folder', None)
compressed = bool(kwargs.pop('compressed', True))
extension = '.pdb'
local_folder = pathPDBFolder()
if local_folder:
local_folder, is_divided = local_folder
if is_divided:
getPath = lambda pdb: join(makePath(join(local_folder, pdb[1:3])),
'pdb' + pdb + '.pdb.gz')
else:
getPath = lambda pdb: join(local_folder, pdb + '.pdb.gz')
if output_folder is None:
second = lambda filename, pdb: filename
else:
if compressed:
second = lambda filename, pdb: (copyFile(filename,
join(output_folder, pdb + extension + '.gz')))
else:
second = lambda filename, pdb: gunzip(filename,
join(output_folder, pdb + extension))
else:
if output_folder is None:
output_folder = getcwd()
if compressed:
getPath = lambda pdb: join(output_folder, pdb + extension + '.gz')
second = lambda filename, pdb: filename
else:
getPath = lambda pdb: join(output_folder, pdb + extension)
second = lambda filename, pdb: gunzip(getPath(pdb), getPath(pdb))
getURL = WWPDB_HTTP_URL[wwPDBServer() or 'us']
success = 0
failure = 0
filenames = []
for pdb in identifiers:
if pdb is None:
filenames.append(None)
continue
try:
handle = openURL(getURL(pdb))
except Exception as err:
LOGGER.warn('{0} download failed ({1}).'.format(pdb, str(err)))
failure += 1
filenames.append(None)
else:
data = handle.read()
if len(data):
filename = getPath(pdb)
with open(filename, 'w+b') as pdbfile:
pdbfile.write(data)
filename = normpath(relpath(second(filename, pdb)))
LOGGER.debug('{0} downloaded ({1})'
.format(pdb, sympath(filename)))
success += 1
filenames.append(filename)
else:
LOGGER.warn('{0} download failed, reason unknown.'
.format(pdb))
failure += 1
filenames.append(None)
LOGGER.debug('PDB download via HTTP completed ({0} downloaded, '
'{1} failed).'.format(success, failure))
if len(identifiers) == 1:
return filenames[0]
else:
return filenames
def fetchPDBviaFTP(*pdb, **kwargs):
"""Retrieve PDB (default), PDBML, mmCIF, or EMD file(s) for specified *pdb*
identifier(s) and return path(s). Downloaded files will be stored in
local PDB folder, if one is set using :meth:`.pathPDBFolder`, and copied
into *folder*, if specified by the user. If no destination folder is
specified, files will be saved in the current working directory. If
*compressed* is **False**, decompressed files will be copied into
*folder*. *format* keyword argument can be used to retrieve
`PDBML <http://pdbml.pdb.org/>`_, `mmCIF <http://mmcif.pdb.org/>`_
and `PDBML <ftp://ftp.wwpdb.org/pub/emdb/doc/Map-format/current/EMDB_map_format.pdf>`_
files: ``format='cif'`` will fetch an mmCIF file, ``format='emd'`` will fetch an EMD file,
and ``format='xml'`` will fetch a PDBML file.
If PDBML header file is desired, ``noatom=True`` argument will do the job."""
if kwargs.get('check', True):
identifiers = checkIdentifiers(*pdb)
else:
identifiers = list(pdb)
output_folder = kwargs.pop('folder', None)
compressed = bool(kwargs.pop('compressed', True))
format = str(kwargs.pop('format', 'pdb')).lower()
noatom = bool(kwargs.pop('noatom', False))
if format == 'pdb':
ftp_divided = 'pdb/data/structures/divided/pdb'
ftp_pdbext = '.ent.gz'
ftp_prefix = 'pdb'
extension = '.pdb'
elif format == 'xml':
if noatom:
ftp_divided = 'pdb/data/structures/divided/XML-noatom'
ftp_pdbext = '-noatom.xml.gz'
extension = '-noatom.xml'
else:
ftp_divided = 'pdb/data/structures/divided/XML'
ftp_pdbext = '.xml.gz'
extension = '.xml'
ftp_prefix = ''
elif format == 'cif':
ftp_divided = 'pdb/data/structures/divided/mmCIF'
ftp_pdbext = '.cif.gz'
ftp_prefix = ''
extension = '.cif'
elif format == 'emd' or format == 'map':
ftp_divided = 'emdb/structures'
ftp_pdbext = '.map.gz'
ftp_prefix = 'emd_'
extension = '.map'
else:
raise ValueError(repr(format) + ' is not valid format')
local_folder = pathPDBFolder()
if format == 'pdb' and local_folder:
local_folder, is_divided = local_folder
if is_divided:
getPath = lambda pdb: join(makePath(join(local_folder, pdb[1:3])),
'pdb' + pdb + '.pdb.gz')
else:
getPath = lambda pdb: join(local_folder, pdb + '.pdb.gz')
if output_folder is None:
second = lambda filename, pdb: filename
else:
if compressed:
second = lambda filename, pdb: (copyFile(filename,
join(output_folder, pdb + extension + '.gz')))
else:
second = lambda filename, pdb: gunzip(filename,
join(output_folder, pdb + extension))
else:
if output_folder is None:
output_folder = getcwd()
if compressed:
getPath = lambda pdb: join(output_folder, pdb + extension + '.gz')
second = lambda filename, pdb: filename
else:
getPath = lambda pdb: join(output_folder, pdb + extension)
second = lambda filename, pdb: gunzip(getPath(pdb), getPath(pdb))
ftp_name, ftp_host, ftp_path = WWPDB_FTP_SERVERS[wwPDBServer() or 'us']
LOGGER.debug('Connecting wwPDB FTP server {0}.'.format(ftp_name))
from ftplib import FTP
try:
ftp = FTP(ftp_host)
except Exception as error:
raise type(error)('FTP connection problem, potential reason: '
'no internet connectivity')
else:
success = 0
failure = 0
filenames = []
ftp.login('')
for pdb in identifiers:
if pdb is None:
filenames.append(None)
continue
data = []
ftp_fn = ftp_prefix + pdb + ftp_pdbext
try:
ftp.cwd(ftp_path)
ftp.cwd(ftp_divided)
if format == 'emd':
ftp.cwd('EMD-{0}/map'.format(pdb))
else:
ftp.cwd(pdb[1:3])
ftp.retrbinary('RETR ' + ftp_fn, data.append)
except Exception as error:
if ftp_fn in ftp.nlst():
LOGGER.warn('{0} download failed ({1}). It is '
'possible that you do not have rights to '
'download .gz files in the current network.'
.format(pdb, str(error)))
else:
LOGGER.info('{0} download failed. {1} does not exist '
'on {2}.'.format(ftp_fn, pdb, ftp_host))
failure += 1
filenames.append(None)
else:
if len(data):
filename = getPath(pdb)
with open(filename, 'w+b') as pdbfile:
write = pdbfile.write
[write(block) for block in data]
filename = normpath(relpath(second(filename, pdb)))
LOGGER.debug('{0} downloaded ({1})'
.format(pdb, sympath(filename)))
success += 1
filenames.append(filename)
else:
LOGGER.warn('{0} download failed, reason unknown.'
.format(pdb))
failure += 1
filenames.append(None)
ftp.quit()
LOGGER.debug('PDB download via FTP completed ({0} downloaded, '
'{1} failed).'.format(success, failure))
if len(identifiers) == 1:
return filenames[0]
else:
return filenames
