def download_and_install(url, name=DEFAULT_MODULE_NAME, cache=True):
# type: (str, str, bool) -> module
"""Download, prepare and install a compressed tar file from S3 as a module.
SageMaker Python SDK saves the user provided scripts as compressed tar files in S3
https://github.com/aws/sagemaker-python-sdk.
This function downloads this compressed file, transforms it as a module, and installs it.
Args:
name (str): name of the script or module.
url (str): the s3 url of the file.
cache (bool): default True. It will not download and install the module again if it is already installed.
Returns:
(module): the imported module
"""
should_use_cache = cache and exists(name)
if not should_use_cache:
with _files.tmpdir() as tmpdir:
dst = os.path.join(tmpdir, 'tar_file')
s3_download(url, dst)
module_path = os.path.join(tmpdir, 'module_dir')
os.makedirs(module_path)
with tarfile.open(name=dst, mode='r:gz') as t:
t.extractall(path=module_path)
prepare(module_path, name)
install(module_path)
def download_and_install(uri, name=DEFAULT_MODULE_NAME, cache=True):
# type: (str, str, bool) -> None
"""Download, prepare and install a compressed tar file from S3 or local directory as a module.
The SageMaker Python SDK saves the user provided scripts as compressed tar files in S3.
This function downloads this compressed file and, if provided, transforms it
into a module before installing it.
This method is the predecessor of
:meth:`~sagemaker_containers.beta.framework.files.download_and_extract`
and has been kept for backward-compatibility purposes.
Args:
name (str): name of the script or module.
uri (str): the location of the module.
cache (bool): defaults to True. It will not download and install the module again if it is
already installed.
"""
should_use_cache = cache and exists(name)
if not should_use_cache:
with _files.tmpdir() as tmpdir:
dst = os.path.join(tmpdir, "tar_file")
_files.download_and_extract(uri, dst)
module_path = os.path.join(tmpdir, "module_dir")
os.makedirs(module_path)
prepare(module_path, name)
install(module_path)
def upload(self): # type: () -> UserModule
with _files.tmpdir() as tmpdir:
tar_name = os.path.join(tmpdir, 'sourcedir.tar.gz')
with tarfile.open(tar_name, mode='w:gz') as tar:
for _file in self._files:
name = os.path.join(tmpdir, _file.name)
with open(name, 'w+') as f:
if isinstance(_file.data, six.string_types):
data = _file.data
else:
data = '\n'.join(_file.data)
f.write(data)
tar.add(name=name, arcname=_file.name)
self._s3.Object(self.bucket, self.key).upload_file(tar_name)
return self
def test_tmpdir_with_args(rmtree, mkdtemp):
with _files.tmpdir('suffix', 'prefix', '/tmp'):
mkdtemp.assert_called_with(dir='/tmp',
prefix='prefix',
suffix='suffix')
rmtree.assert_called()
def test_tmpdir(rmtree, mkdtemp):
with _files.tmpdir():
mkdtemp.assert_called()
rmtree.assert_called()
def test_tmpdir_with_args(rmtree, mkdtemp):
with _files.tmpdir("suffix", "prefix", "/tmp"):
mkdtemp.assert_called_with(dir="/tmp", prefix="prefix", suffix="suffix")
rmtree.assert_called()
def upload(self): # type: () -> UserModule
with _files.tmpdir() as tmpdir:
tar_name = self.create_tar(dir_path=tmpdir)
self._s3.Object(self.bucket, self.key).upload_file(tar_name)
return self
