def download_data(output='data/noisy_mnist.npz'):
"""
Downloads and extracts Noisy MNIST dataset (~286MB)
The resulting .npz file contains train, valid, and test
image pairs along with their labels in arrays train_digit,
valid_digit, and test_difit
Parameters:
output: path to the output .npz file
"""
with tempfile.NamedTemporaryFile() as view1, \
tempfile.NamedTemporaryFile() as view2:
logger.info("[Step 1/3] Downloading images x (rotated)")
_download_url_to_file(__VIEW1, view1.name, None, True)
logger.info("[Step 2/3] Downloading images y (rotated and noisy)")
_download_url_to_file(__VIEW2, view2.name, None, True)
logger.info("[Step 3/3] Preparing final file")
img_x = read_data(view1.name)
img_y = read_data(view2.name)
train = np.asarray([img_x[0][0], img_y[0][0]])
valid = np.asarray([img_x[1][0], img_y[1][0]])
test = np.asarray([img_x[2][0], img_y[2][0]])
train_digit = img_x[0][1]
valid_digit = img_x[1][1]
test_digit = img_x[2][1]
assert np.all(img_y[0][1] == train_digit)
assert np.all(img_y[1][1] == valid_digit)
assert np.all(img_y[2][1] == test_digit)
np.savez_compressed(output,
train=train,
valid=valid,
test=test,
train_digit=train_digit,
valid_digit=valid_digit,
test_digit=test_digit)
def load_params_from_url(url, param_dir=None, progress=True):
r"""
Loads the PCA params using the syntax from https://github.com/pytorch/pytorch/blob/master/torch/hub.py,
except doesn't serialize using torch.load, simply provides files as numpy format.
"""
if os.getenv("TORCH_MODEL_ZOO"):
warnings.warn(
"TORCH_MODEL_ZOO is deprecated, please use env TORCH_HOME instead")
if param_dir is None:
torch_home = hub._get_torch_home()
param_dir = os.path.join(torch_home, "params")
try:
os.makedirs(param_dir)
except OSError as e:
if e.errno == errno.EEXIST:
pass
else:
# Unexpected OSError, re-raise.
raise
parts = urlparse(url)
filename = os.path.basename(parts.path)
cached_file = os.path.join(param_dir, filename)
if not os.path.exists(cached_file):
sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file))
hash_prefix = hub.HASH_REGEX.search(filename).group(1)
hub._download_url_to_file(url,
cached_file,
hash_prefix,
progress=progress)
return np.load(cached_file)
def cache_url(url, model_dir=None, progress=True):
"""
Loads the Torch serialized object at the given URL.
Args:
url:
model_dir:
progress:
Returns:
"""
if model_dir is None:
torch_home = os.path.expanduser(os.getenv("TORCH_HOME", "~/.torch"))
model_dir = os.getenv("TORCH_MODEL_ZOO", os.path.join(torch_home, "models"))
if not os.path.exists(model_dir):
os.makedirs(model_dir)
parts = urlparse(url)
filename = os.path.basename(parts.path)
if filename == "model_final.pkl":
filename = parts.path.replace("/", "_")
cached_file = os.path.join(model_dir, filename)
if not os.path.exists(cached_file) and is_main_process():
sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file))
hash_prefix = HASH_REGEX.search(filename)
if hash_prefix is not None:
hash_prefix = hash_prefix.group(1)
if len(hash_prefix) < 6:
hash_prefix = None
_download_url_to_file(url, cached_file, hash_prefix, progress=progress)
synchronize()
return cached_file
def load_state_dict_from_url(url,
model_dir=None,
map_location=None,
progress=True):
r"""Loads the Torch serialized object at the given URL.
If the object is already present in `model_dir`, it's deserialized and
returned. The filename part of the URL should follow the naming convention
``filename-<sha256>.ext`` where ``<sha256>`` is the first eight or more
digits of the SHA256 hash of the contents of the file. The hash is used to
ensure unique names and to verify the contents of the file.
The default value of `model_dir` is ``$TORCH_HOME/checkpoints`` where
environment variable ``$TORCH_HOME`` defaults to ``$XDG_CACHE_HOME/torch``.
``$XDG_CACHE_HOME`` follows the X Design Group specification of the Linux
filesytem layout, with a default value ``~/.cache`` if not set.
Args:
url (string): URL of the object to download
model_dir (string, optional): directory in which to save the object
map_location (optional): a function or a dict specifying how to remap storage locations (see torch.load)
progress (bool, optional): whether or not to display a progress bar to stderr
Example:
>>> state_dict = torch.hub.load_state_dict_from_url('https://s3.amazonaws.com/pytorch/models/resnet18-5c106cde.pth')
"""
# Issue warning to move data if old env is set
if os.getenv('TORCH_MODEL_ZOO'):
warnings.warn(
'TORCH_MODEL_ZOO is deprecated, please use env TORCH_HOME instead')
if model_dir is None:
torch_home = _get_torch_home()
model_dir = os.path.join(torch_home, 'checkpoints')
try:
os.makedirs(model_dir)
except OSError as e:
if e.errno == errno.EEXIST:
# Directory already exists, ignore.
pass
else:
# Unexpected OSError, re-raise.
raise
parts = urlparse(url)
filename = os.path.basename(parts.path)
cached_file = os.path.join(model_dir, filename)
if not os.path.exists(cached_file):
sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file))
hash_prefix = HASH_REGEX.search(filename)
if hash_prefix:
hash_prefix = hash_prefix.group(1)
_download_url_to_file(url, cached_file, hash_prefix, progress=progress)
return torch.load(cached_file, map_location=map_location)
def get_file_from_url(url, model_dir=None, progress=True, unzip=True):
"""
Download model from url and return path to downloaded model.
Args:
url (string): URL of the object to download
model_dir (string, optional): directory in which to save the object
progress (bool, optional): whether or not to display a progress bar to stderr.
Default: True
unzip (bool, optional): whether to use unzip. Don't use with TorchScript models!
Default: True
Returns: downloaded model path
"""
if model_dir is None:
torch_home = _get_torch_home()
model_dir = os.path.join(torch_home, "checkpoints")
try:
os.makedirs(model_dir)
except OSError as e:
if e.errno == errno.EEXIST:
# Directory already exists, ignore.
pass
else:
# Unexpected OSError, re-raise.
raise
parts = urlparse(url)
filename = os.path.basename(parts.path)
cached_file = os.path.join(model_dir, filename)
if not os.path.exists(cached_file):
logger.info(f"Downloading: {url} to {cached_file}")
_download_url_to_file(url, cached_file, None, progress=progress)
# Note: extractall() defaults to overwrite file if exists. No need to clean up beforehand.
# We deliberately don't handle tarfile here since our legacy serialization format was in tar.
# E.g. resnet18-5c106cde.pth which is widely used.
if zipfile.is_zipfile(cached_file) and unzip:
with zipfile.ZipFile(cached_file) as cached_zipfile:
members = cached_zipfile.infolist()
if len(members) != 1:
raise RuntimeError(
"Only one file(not dir) is allowed in the zipfile")
cached_zipfile.extractall(model_dir)
extraced_name = members[0].filename
cached_file = os.path.join(model_dir, extraced_name)
return cached_file
def cache_url(url, model_dir=None, progress=True):
r"""Loads the Torch serialized object at the given URL.
If the object is already present in `model_dir`, it's deserialized and
returned. The filename part of the URL should follow the naming convention
``filename-<sha256>.ext`` where ``<sha256>`` is the first eight or more
digits of the SHA256 hash of the contents of the file. The hash is used to
ensure unique names and to verify the contents of the file.
The default value of `model_dir` is ``$TORCH_HOME/models`` where
``$TORCH_HOME`` defaults to ``~/.torch``. The default directory can be
overridden with the ``$TORCH_MODEL_ZOO`` environment variable.
Args:
url (string): URL of the object to download
model_dir (string, optional): directory in which to save the object
progress (bool, optional): whether or not to display a progress bar to stderr
Example:
>>> cached_file = sampling_free.utils.model_zoo.cache_url('https://s3.amazonaws.com/pytorch/models/resnet18-5c106cde.pth')
"""
if model_dir is None:
torch_home = os.path.expanduser(os.getenv('TORCH_HOME', '~/.torch'))
model_dir = os.getenv('TORCH_MODEL_ZOO',
os.path.join(torch_home, 'models'))
if not os.path.exists(model_dir):
os.makedirs(model_dir)
parts = urlparse(url)
if parts.fragment != "":
filename = parts.fragment
else:
filename = os.path.basename(parts.path)
if filename == "model_final.pkl":
# workaround as pre-trained Caffe2 models from Detectron have all the same filename
# so make the full path the filename by replacing / with _
filename = parts.path.replace("/", "_")
cached_file = os.path.join(model_dir, filename)
if not os.path.exists(cached_file) and is_main_process():
sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file))
hash_prefix = HASH_REGEX.search(filename)
if hash_prefix is not None:
hash_prefix = hash_prefix.group(1)
# workaround: Caffe2 models don't have a hash, but follow the R-50 convention,
# which matches the hash PyTorch uses. So we skip the hash matching
# if the hash_prefix is less than 6 characters
if len(hash_prefix) < 6:
hash_prefix = None
_download_url_to_file(url, cached_file, hash_prefix, progress=progress)
synchronize()
return cached_file
def load_url(url,
model_dir=None,
map_location=None,
progress=True,
hash_prefix=None,
filename=None):
if model_dir is None:
torch_home = os.path.expanduser(os.getenv('TORCH_HOME', '~/.torch'))
model_dir = os.getenv('TORCH_MODEL_ZOO',
os.path.join(torch_home, 'models'))
if not os.path.exists(model_dir):
os.makedirs(model_dir)
parts = urlparse(url)
if filename is None: filename = os.path.basename(parts.path)
cached_file = os.path.join(model_dir, filename)
if not os.path.exists(cached_file):
sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file))
if hash_prefix is None:
hash_prefix = HASH_REGEX.search(filename).group(1)
_download_url_to_file(url, cached_file, hash_prefix, progress=progress)
return torch.load(cached_file, map_location=map_location)
import os
import zipfile
#from torch.utils.model_zoo import _download_url_to_file
from torch.hub import _download_url_to_file
def unzip(source_filename, dest_dir):
with zipfile.ZipFile(source_filename) as zf:
zf.extractall(path=dest_dir)
if __name__ == '__main__':
_download_url_to_file(
'https://www.dropbox.com/s/lrvwfehqdcxoza8/saved_models.zip?dl=1',
'saved_models.zip', None, True)
unzip('saved_models.zip', '.')
