def download_and_install(url, name=DEFAULT_MODULE_NAME, cache=True): # type: (str, str, bool) -> module """Download, prepare and install a compressed tar file from S3 as a module. SageMaker Python SDK saves the user provided scripts as compressed tar files in S3 https://github.com/aws/sagemaker-python-sdk. This function downloads this compressed file, transforms it as a module, and installs it. Args: name (str): name of the script or module. url (str): the s3 url of the file. cache (bool): default True. It will not download and install the module again if it is already installed. Returns: (module): the imported module """ should_use_cache = cache and exists(name) if not should_use_cache: with _files.tmpdir() as tmpdir: dst = os.path.join(tmpdir, 'tar_file') s3_download(url, dst) module_path = os.path.join(tmpdir, 'module_dir') os.makedirs(module_path) with tarfile.open(name=dst, mode='r:gz') as t: t.extractall(path=module_path) prepare(module_path, name) install(module_path)
def download_and_install(uri, name=DEFAULT_MODULE_NAME, cache=True): # type: (str, str, bool) -> None """Download, prepare and install a compressed tar file from S3 or local directory as a module. The SageMaker Python SDK saves the user provided scripts as compressed tar files in S3. This function downloads this compressed file and, if provided, transforms it into a module before installing it. This method is the predecessor of :meth:`~sagemaker_containers.beta.framework.files.download_and_extract` and has been kept for backward-compatibility purposes. Args: name (str): name of the script or module. uri (str): the location of the module. cache (bool): defaults to True. It will not download and install the module again if it is already installed. """ should_use_cache = cache and exists(name) if not should_use_cache: with _files.tmpdir() as tmpdir: dst = os.path.join(tmpdir, "tar_file") _files.download_and_extract(uri, dst) module_path = os.path.join(tmpdir, "module_dir") os.makedirs(module_path) prepare(module_path, name) install(module_path)
def upload(self): # type: () -> UserModule with _files.tmpdir() as tmpdir: tar_name = os.path.join(tmpdir, 'sourcedir.tar.gz') with tarfile.open(tar_name, mode='w:gz') as tar: for _file in self._files: name = os.path.join(tmpdir, _file.name) with open(name, 'w+') as f: if isinstance(_file.data, six.string_types): data = _file.data else: data = '\n'.join(_file.data) f.write(data) tar.add(name=name, arcname=_file.name) self._s3.Object(self.bucket, self.key).upload_file(tar_name) return self
def test_tmpdir_with_args(rmtree, mkdtemp): with _files.tmpdir('suffix', 'prefix', '/tmp'): mkdtemp.assert_called_with(dir='/tmp', prefix='prefix', suffix='suffix') rmtree.assert_called()
def test_tmpdir(rmtree, mkdtemp): with _files.tmpdir(): mkdtemp.assert_called() rmtree.assert_called()
def test_tmpdir_with_args(rmtree, mkdtemp): with _files.tmpdir("suffix", "prefix", "/tmp"): mkdtemp.assert_called_with(dir="/tmp", prefix="prefix", suffix="suffix") rmtree.assert_called()
def upload(self): # type: () -> UserModule with _files.tmpdir() as tmpdir: tar_name = self.create_tar(dir_path=tmpdir) self._s3.Object(self.bucket, self.key).upload_file(tar_name) return self