def test_download_file_no_collision(self, experiment_run, dir_and_files, in_tempdir): source_dirpath, _ = dir_and_files key = "artifact" # create archive and move into cwd so it's deleted on teardown filepath = os.path.abspath("archive.zip") temp_zip = _artifact_utils.zip_dir(source_dirpath) os.rename(temp_zip.name, filepath) # upload and download file experiment_run.log_artifact(key, filepath) download_url = experiment_run._get_url_for_artifact(key, "GET").url response = requests.get(download_url) downloaded_filepath = _request_utils.download_file( response, filepath, overwrite_ok=False, ) downloaded_filepath = os.path.abspath(downloaded_filepath) # different names assert filepath != downloaded_filepath # contents match assert filecmp.cmp(filepath, downloaded_filepath)
def test_download_arbitrary_zip(self, experiment_run, dir_and_files, strs, in_tempdir): """Model that was originally a ZIP is not unpacked on download.""" model_dir, _ = dir_and_files upload_path, download_path = strs[:2] # zip `model_dir` into `upload_path` with open(upload_path, 'wb') as f: shutil.copyfileobj( _artifact_utils.zip_dir(model_dir), f, ) experiment_run.log_model(upload_path) experiment_run.download_model(download_path) assert zipfile.is_zipfile(download_path) assert filecmp.cmp(upload_path, download_path)
def test_download_arbitrary_zip(self, model_version, dir_and_files, strs, in_tempdir): """Model that was originally a ZIP is not unpacked on download.""" model_dir, _ = dir_and_files upload_path, download_path = strs[:2] # zip `model_dir` into `upload_path` with open(upload_path, "wb") as f: shutil.copyfileobj( _artifact_utils.zip_dir(model_dir), f, ) model_version.log_model(upload_path) returned_path = model_version.download_model(download_path) assert returned_path == os.path.abspath(download_path) assert zipfile.is_zipfile(download_path) assert filecmp.cmp(upload_path, download_path)
def log_artifact(self, key, artifact, overwrite=False, _extension=None): """ Logs an artifact to this Model Version. .. note:: The following artifact keys are reserved for internal use within the Verta system: - ``"custom_modules"`` - ``"model"`` - ``"model.pkl"`` - ``"model_api.json"`` - ``"requirements.txt"`` - ``"train_data"`` - ``"tf_saved_model"`` - ``"setup_script"`` Parameters ---------- key : str Name of the artifact. artifact : str or file-like or object Artifact or some representation thereof. - If str, then it will be interpreted as a filesystem path, its contents read as bytes, and uploaded as an artifact. If it is a directory path, its contents will be zipped. - If file-like, then the contents will be read as bytes and uploaded as an artifact. - Otherwise, the object will be serialized and uploaded as an artifact. overwrite : bool, default False Whether to allow overwriting an existing artifact with key `key`. """ # TODO: should validate keys, but can't here because this public # method is also used to log internal artifacts # _artifact_utils.validate_key(key) if key == self._MODEL_KEY: raise ValueError('the key "{}" is reserved for model;' " consider using log_model() instead".format( self._MODEL_KEY)) self._fetch_with_no_cache() same_key_ind = -1 for i in range(len(self._msg.artifacts)): if self._msg.artifacts[i].key == key: if not overwrite: raise ValueError( "The key has been set; consider setting overwrite=True" ) else: same_key_ind = i break artifact_type = _CommonCommonService.ArtifactTypeEnum.BLOB if isinstance(artifact, six.string_types): if os.path.isdir(artifact): # zip dirpath artifact = _artifact_utils.zip_dir(artifact) else: # open filepath artifact = open(artifact, "rb") artifact_stream, method = _artifact_utils.ensure_bytestream(artifact) artifact_msg = self._create_artifact_msg( key, artifact_stream, artifact_type=artifact_type, method=method, extension=_extension, ) if same_key_ind == -1: self._msg.artifacts.append(artifact_msg) else: self._msg.artifacts[same_key_ind].CopyFrom(artifact_msg) self._update(self._msg, method="PUT") self._upload_artifact(key, artifact_stream, artifact_type=artifact_type)