def _image_from_url(url):
'''a method for loading images from urls by first saving them locally'''
filename = os.path.basename(url)
if not os.path.exists(filename):
if '://' not in url: url = 'http://' + url
_urlretrieve(url, filename)
image, filename = _image_from_file(filename)
return image, filename
def urlretrieve(url, filename, reporthook, sha256sum):
try:
_urlretrieve(url, filename, reporthook)
if not validate_sha256(filename, sha256sum):
raise DownloadError("Corrupted download, the sha256 doesn't match")
except BaseException:
os.unlink(filename)
raise
def _image_from_url(url):
'''a method for loading images from urls by first saving them locally'''
filename = os.path.basename(url)
if not os.path.exists(filename):
if '://' not in url: url = 'http://'+url
_urlretrieve(url, filename)
image, filename =_image_from_file(filename)
return image, filename
def urlretrieve(url, filename, reporthook, sha256sum):
try:
_urlretrieve(url, filename, reporthook)
if not validate_sha256(filename, sha256sum):
raise DownloadError(
"Corrupted download, the sha256 doesn't match")
except BaseException:
os.unlink(filename)
raise
def _image_from_url(url):
"""a method for loading images from urls by first saving them locally"""
filename = os.path.basename(url)
if not os.path.exists(filename):
if "://" not in url:
url = "http://" + url
_urlretrieve(url, filename)
image, filename = _image_from_file(filename)
return image, filename
def load_sound(url_or_filename):
'''Reads a sound file from a given filename or url'''
if url_or_filename in _known_images: return _known_sounds[url_or_filename]
if not os.path.exists(url_or_filename):
filename = os.path.basename(url_or_filename)
if not os.path.exists(filename):
_urlretrieve(url_or_filename, filename)
url_or_filename = filename
sound = pygame.mixer.Sound(url_or_filename)
_known_sounds[url_or_filename] = sound
return sound
def load_sound(url_or_filename):
'''Reads a sound file from a given filename or url'''
if url_or_filename in _known_images: return _known_sounds[url_or_filename]
if not os.path.exists(url_or_filename):
filename = os.path.basename(url_or_filename)
if not os.path.exists(filename):
_urlretrieve(url_or_filename, filename)
url_or_filename = filename
sound = pygame.mixer.Sound(url_or_filename)
_known_sounds[url_or_filename] = sound
return sound
def urlretrieve(url, filename=None):
'''Download a file to disk
'''
if filename is None:
filename = os.path.basename(url)
debug('DOWNLOAD', '%s from %s', name, os.path.dirname(url))
else:
debug('DOWNLOAD', '%s as %s', url, filename)
if _urlretrieve is None:
data = urllib2.urlopen(url).read()
with open(filename, 'w') as f:
f.write(data)
else:
_urlretrieve(url, filename)
def open_pdb(pdbid, pdb_url=None):
"""Make a local copy of an online pdb file and return a file handle."""
if pdb_url is None:
pdb_url = default_pdb_url
url = pdb_url % pdbid
fn, header = _urlretrieve(url)
return open(fn)
def load_dataset(
name: Optional[str] = None,
cache: bool = True,
cache_dir: str = _default_cache_dir,
github_url: str = "https://github.com/pangeo-data/climpred-data",
branch: str = "master",
extension: Optional[str] = None,
proxy_dict: Optional[Dict[str, str]] = None,
**kws,
) -> xr.Dataset:
"""Load example data or a mask from an online repository.
Args:
name: Name of the netcdf file containing the
dataset, without the ``.nc`` extension. If ``None``, this function
prints out the available datasets to import.
cache: If ``True``, cache data locally for use on later calls.
cache_dir: The directory in which to search for and cache the data.
github_url: Github repository where the data is stored.
branch: The git branch to download from.
extension: Subfolder within the repository where the data is stored.
proxy_dict: Dictionary with keys as either "http" or "https" and values as the
proxy server. This is useful if you are on a work computer behind a
firewall and need to use a proxy out to download data.
kws: Keywords passed to :py:meth:`~xarray.open_dataset`.
Returns:
The desired :py:class:`xarray.Dataset`
Examples:
>>> from climpred.tutorial import load_dataset
>>> proxy_dict = {"http": "127.0.0.1"}
>>> ds = load_dataset("FOSI-SST", cache=False, proxy_dict=proxy_dict)
"""
if name is None:
return _get_datasets()
if proxy_dict is not None:
_initialize_proxy(proxy_dict)
# https://stackoverflow.com/questions/541390/extracting-extension-from-
# filename-in-python
# Allows for generalized file extensions.
name, ext = _os.path.splitext(name)
if not ext.endswith(".nc"):
ext += ".nc"
# use aliases
if name in FILE_ALIAS_DICT.keys():
name = FILE_ALIAS_DICT[name]
longdir = _os.path.expanduser(cache_dir)
fullname = name + ext
localfile = _os.sep.join((longdir, fullname))
md5name = name + ".md5"
md5file = _os.sep.join((longdir, md5name))
if not _os.path.exists(localfile):
# This will always leave this directory on disk.
# May want to add an option to remove it.
if not _os.path.isdir(longdir):
_os.mkdir(longdir)
if extension is not None:
url = "/".join((github_url, "raw", branch, extension, fullname))
_urlretrieve(url, localfile)
url = "/".join((github_url, "raw", branch, extension, md5name))
_urlretrieve(url, md5file)
else:
url = "/".join((github_url, "raw", branch, fullname))
_urlretrieve(url, localfile)
url = "/".join((github_url, "raw", branch, md5name))
_urlretrieve(url, md5file)
localmd5 = _file_md5_checksum(localfile)
with open(md5file, "r") as f:
remotemd5 = f.read()
if localmd5 != remotemd5:
_os.remove(localfile)
msg = """
Try downloading the file again. There was a confliction between
your local .md5 file compared to the one in the remote repository,
so the local copy has been removed to resolve the issue.
"""
raise IOError(msg)
ds = _open_dataset(localfile, **kws)
if not cache:
ds = ds.load()
_os.remove(localfile)
return ds
def urlretrieve(url, output_path):
with DownloadProgressBar(unit='B', unit_scale=True, miniters=1,
desc=url.split('/')[-1]) as t:
_urlretrieve(url, filename=output_path, reporthook=t.update_to)
def open_dataset(name,
cache=True,
cache_dir=_default_cache_dir,
github_url='https://github.com/bradyrx/climpred',
branch='master',
extension='sample_data/prediction',
**kws):
"""Load example data or a mask from an online repository.
This is a function from `xarray.tutorial` to load an online dataset
with minimal package imports. I am copying it here because it looks like
it will soon be deprecated. Also, I've added the ability to point to
data files that are not in the main folder of the repo (i.e., they are
in subfolders).
Note that this requires an md5 file to be loaded. Check the github
repo bradyrx/climdata for a python script that converts .nc files into
md5 files.
Args:
name: (str) Name of the netcdf file containing the dataset, without
the .nc extension.
cache_dir: (str, optional) The directory in which to search
for and cache the data.
cache: (bool, optional) If true, cache data locally for use on later
calls.
github_url: (str, optional) Github repository where the data is stored.
branch: (str, optional) The git branch to download from.
extension: (str, optional) Subfolder within the repository where the
data is stored.
kws: (dict, optional) Keywords passed to xarray.open_dataset
Returns:
The desired xarray dataset.
"""
if name.endswith('.nc'):
name = name[:-3]
# use aliases
if name in file_alias_dict.keys():
name = file_alias_dict[name]
longdir = _os.path.expanduser(cache_dir)
fullname = name + '.nc'
localfile = _os.sep.join((longdir, fullname))
md5name = name + '.md5'
md5file = _os.sep.join((longdir, md5name))
if not _os.path.exists(localfile):
# This will always leave this directory on disk.
# May want to add an option to remove it.
if not _os.path.isdir(longdir):
_os.mkdir(longdir)
if extension is not None:
url = '/'.join((github_url, 'raw', branch, extension, fullname))
_urlretrieve(url, localfile)
url = '/'.join((github_url, 'raw', branch, extension, md5name))
_urlretrieve(url, md5file)
else:
url = '/'.join((github_url, 'raw', branch, fullname))
_urlretrieve(url, localfile)
url = '/'.join((github_url, 'raw', branch, md5name))
_urlretrieve(url, md5file)
localmd5 = _file_md5_checksum(localfile)
with open(md5file, 'r') as f:
remotemd5 = f.read()
if localmd5 != remotemd5:
_os.remove(localfile)
msg = """
MD5 checksum does not match, try downloading dataset again.
"""
raise IOError(msg)
ds = _open_dataset(localfile, **kws)
if not cache:
ds = ds.load()
_os.remove(localfile)
return ds
def load_dataset(
name=None,
cache=True,
cache_dir=_default_cache_dir,
github_url='https://github.com/bradyrx/climpred-data',
branch='master',
extension=None,
proxy_dict=None,
**kws,
):
"""Load example data or a mask from an online repository.
Args:
name: (str, default None) Name of the netcdf file containing the
dataset, without the .nc extension. If None, this function
prints out the available datasets to import.
cache_dir: (str, optional) The directory in which to search
for and cache the data.
cache: (bool, optional) If True, cache data locally for use on later
calls.
github_url: (str, optional) Github repository where the data is stored.
branch: (str, optional) The git branch to download from.
extension: (str, optional) Subfolder within the repository where the
data is stored.
proxy_dict: (dict, optional) Dictionary with keys as either 'http' or
'https' and values as the proxy server. This is useful
if you are on a work computer behind a firewall and need
to use a proxy out to download data.
kws: (dict, optional) Keywords passed to xarray.open_dataset
Returns:
The desired xarray dataset.
Examples:
>>> from climpred.tutorial import load_dataset()
>>> proxy_dict = {'http': '127.0.0.1'}
>>> ds = load_dataset('FOSI-SST', cache=False, proxy_dict=proxy_dict)
"""
if name is None:
return _get_datasets()
if proxy_dict is not None:
_initialize_proxy(proxy_dict)
# https://stackoverflow.com/questions/541390/extracting-extension-from-
# filename-in-python
# Allows for generalized file extensions.
name, ext = _os.path.splitext(name)
if not ext.endswith('.nc'):
ext += '.nc'
# use aliases
if name in FILE_ALIAS_DICT.keys():
name = FILE_ALIAS_DICT[name]
longdir = _os.path.expanduser(cache_dir)
fullname = name + ext
localfile = _os.sep.join((longdir, fullname))
md5name = name + '.md5'
md5file = _os.sep.join((longdir, md5name))
if not _os.path.exists(localfile):
# This will always leave this directory on disk.
# May want to add an option to remove it.
if not _os.path.isdir(longdir):
_os.mkdir(longdir)
if extension is not None:
url = '/'.join((github_url, 'raw', branch, extension, fullname))
_urlretrieve(url, localfile)
url = '/'.join((github_url, 'raw', branch, extension, md5name))
_urlretrieve(url, md5file)
else:
url = '/'.join((github_url, 'raw', branch, fullname))
_urlretrieve(url, localfile)
url = '/'.join((github_url, 'raw', branch, md5name))
_urlretrieve(url, md5file)
localmd5 = _file_md5_checksum(localfile)
with open(md5file, 'r') as f:
remotemd5 = f.read()
if localmd5 != remotemd5:
_os.remove(localfile)
msg = """
Try downloading the file again. There was a confliction between
your local .md5 file compared to the one in the remote repository,
so the local copy has been removed to resolve the issue.
"""
raise IOError(msg)
ds = _open_dataset(localfile, **kws)
if not cache:
ds = ds.load()
_os.remove(localfile)
return ds
def main():
"""Extract a SCOP domain's ATOM and HETATOM records from a PDB file."""
try:
opts, args = getopt.getopt(
sys.argv[1:], "hp:o:i:",
["help", "usage", "pdb=", "output=", "input="])
except getopt.GetoptError:
# show help information and exit:
usage()
sys.exit(2)
input = None
in_handle = None
output = None
pdb_url = None
cla_url = None
raf_url = None
for o, a in opts:
if o in ("-h", "--help", "--usage"):
usage()
sys.exit()
elif o in ("-o", "--output"):
output = a
elif o in ("-i", "--input"):
input = a
elif o in ("-p", "--pdb"):
pdb_url = a
if len(args) < 2:
sys.stderr.write(
"Not enough arguments. Try --help for more details.\n")
sys.exit(2)
raf_url = args[0]
cla_url = args[1]
(raf_filename, headers) = _urlretrieve(raf_url)
seqMapIndex = Raf.SeqMapIndex(raf_filename)
(cla_filename, headers) = _urlretrieve(cla_url)
claIndex = Cla.Index(cla_filename)
if input is None:
sids = args[2:]
elif input == "-":
sids = sys.stdin
else:
in_handle = open(input)
sids = in_handle
try:
for sid in sids:
if not sid or sid[0:1] == "#":
continue
id = sid[0:7]
pdbid = id[1:5]
s = pdbid[0:1]
if s == "0" or s == "s":
sys.stderr.write("No coordinates for domain %s\n" % id)
continue
if output is None:
filename = id + ".ent"
out_handle = open(filename, "w+")
elif output == "-":
out_handle = sys.stdout
else:
out_handle = open(output, "w+")
try:
try:
claRec = claIndex[id]
residues = claRec.residues
seqMap = seqMapIndex.getSeqMap(residues)
pdbid = residues.pdbid
f = open_pdb(pdbid, pdb_url)
try:
seqMap.getAtoms(f, out_handle)
finally:
f.close()
except (IOError, KeyError, RuntimeError) as e:
sys.stderr.write("I cannot do SCOP domain %s : %s\n" %
(id, e))
finally:
out_handle.close()
finally:
if in_handle is not None:
in_handle.close()
def get_seqres_file(self, savefile="pdb_seqres.txt"):
"""Retrieve and save a (big) file containing all the sequences of PDB entries."""
if self._verbose:
print("Retrieving sequence file (takes over 110 MB).")
url = self.pdb_server + "/pub/pdb/derived_data/pdb_seqres.txt"
_urlretrieve(url, savefile)
def retrieve_pdb_file(self,
pdb_code,
obsolete=False,
pdir=None,
file_format=None,
overwrite=False):
"""Fetch PDB structure file from PDB server, and store it locally.
The PDB structure's file name is returned as a single string.
If obsolete ``==`` True, the file will be saved in a special file tree.
NOTE. The default download format has changed from PDB to PDBx/mmCif
:param pdb_code: 4-symbols structure Id from PDB (e.g. 3J92).
:type pdb_code: string
:param file_format:
File format. Available options:
* "mmCif" (default, PDBx/mmCif file),
* "pdb" (format PDB),
* "xml" (PDBML/XML format),
* "mmtf" (highly compressed),
* "bundle" (PDB formatted archive for large structure}
:type file_format: string
:param overwrite: if set to True, existing structure files will be overwritten. Default: False
:type overwrite: bool
:param obsolete:
Has a meaning only for obsolete structures. If True, download the obsolete structure
to 'obsolete' folder, otherwise download won't be performed.
This option doesn't work for mmtf format as obsoleted structures aren't stored in mmtf.
Also doesn't have meaning when parameter pdir is specified.
Note: make sure that you are about to download the really obsolete structure.
Trying to download non-obsolete structure into obsolete folder will not work
and you face the "structure doesn't exists" error.
Default: False
:type obsolete: bool
:param pdir: put the file in this directory (default: create a PDB-style directory tree)
:type pdir: string
:return: filename
:rtype: string
"""
# Deprecation warning
file_format = self._print_default_format_warning(file_format)
# Get the compressed PDB structure
code = pdb_code.lower()
archive = {
"pdb": "pdb%s.ent.gz",
"mmCif": "%s.cif.gz",
"xml": "%s.xml.gz",
"mmtf": "%s",
"bundle": "%s-pdb-bundle.tar.gz",
}
archive_fn = archive[file_format] % code
if file_format not in archive.keys():
raise (
"Specified file_format %s doesn't exists or is not supported. Maybe a "
"typo. Please, use one of the following: mmCif, pdb, xml, mmtf, bundle"
% file_format)
if file_format in ("pdb", "mmCif", "xml"):
pdb_dir = "divided" if not obsolete else "obsolete"
file_type = ("pdb" if file_format == "pdb" else
"mmCIF" if file_format == "mmCif" else "XML")
url = self.pdb_server + "/pub/pdb/data/structures/%s/%s/%s/%s" % (
pdb_dir,
file_type,
code[1:3],
archive_fn,
)
elif file_format == "bundle":
url = self.pdb_server + "/pub/pdb/compatible/pdb_bundle/%s/%s/%s" % (
code[1:3],
code,
archive_fn,
)
else:
url = "http://mmtf.rcsb.org/v1.0/full/%s" % code
# Where does the final PDB file get saved?
if pdir is None:
path = self.local_pdb if not obsolete else self.obsolete_pdb
if not self.flat_tree: # Put in PDB-style directory tree
path = os.path.join(path, code[1:3])
else: # Put in specified directory
path = pdir
if not os.access(path, os.F_OK):
os.makedirs(path)
filename = os.path.join(path, archive_fn)
final = {
"pdb": "pdb%s.ent",
"mmCif": "%s.cif",
"xml": "%s.xml",
"mmtf": "%s.mmtf",
"bundle": "%s-pdb-bundle.tar",
}
final_file = os.path.join(path, final[file_format] % code)
# Skip download if the file already exists
if not overwrite:
if os.path.exists(final_file):
if self._verbose:
print("Structure exists: '%s' " % final_file)
return final_file
# Retrieve the file
if self._verbose:
print("Downloading PDB structure '%s'..." % pdb_code)
try:
_urlcleanup()
_urlretrieve(url, filename)
except IOError:
print("Desired structure doesn't exists")
else:
with gzip.open(filename, "rb") as gz:
with open(final_file, "wb") as out:
out.writelines(gz)
os.remove(filename)
return final_file
