def log_setup_script(self, script, overwrite=False): """ Associate a model deployment setup script with this Experiment Run. .. versionadded:: 0.13.8 Parameters ---------- script : str String composed of valid Python code for executing setup steps at the beginning of model deployment. An on-disk file can be passed in using ``open("path/to/file.py", 'r').read()``. overwrite : bool, default False Whether to allow overwriting an existing setup script. Raises ------ SyntaxError If `script` contains invalid Python. """ # validate `script`'s syntax try: ast.parse(script) except SyntaxError as e: # clarify that the syntax error comes from `script`, and propagate details reason = e.args[0] line_no = e.args[1][1] line = script.splitlines()[line_no - 1] six.raise_from( SyntaxError("{} in provided script on line {}:\n{}".format( reason, line_no, line)), e) # convert into bytes for upload script = six.ensure_binary(script) # convert to file-like for `_log_artifact()` script = six.BytesIO(script) self.log_artifact( "setup_script", script, overwrite, "py", )
def get_artifact(self, key): """ Gets the artifact with name `key` from this Model Version. If the artifact was originally logged as just a filesystem path, that path will be returned. Otherwise, bytes representing the artifact object will be returned. Parameters ---------- key : str Name of the artifact. Returns ------- str or object or bytes Path of the artifact, the artifact object, or a bytestream representing the artifact. """ artifact = self._get_artifact( key, _CommonCommonService.ArtifactTypeEnum.BLOB) artifact_stream = six.BytesIO(artifact) torch = importer.maybe_dependency("torch") if torch is not None: try: obj = torch.load(artifact_stream) except: # not something torch can deserialize artifact_stream.seek(0) else: artifact_stream.close() return obj try: obj = pickle.load(artifact_stream) except: # not something pickle can deserialize artifact_stream.seek(0) else: artifact_stream.close() return obj return artifact_stream
def _custom_modules_as_artifact(self, paths=None): if isinstance(paths, six.string_types): paths = [paths] # If we include a path that is actually a module, then we _must_ add its parent to the # adjusted sys.path in the end so that we can re-import with the same name. forced_local_sys_paths = [] if paths is not None: new_paths = [] for p in paths: abspath = os.path.abspath(os.path.expanduser(p)) if os.path.exists(abspath): new_paths.append(abspath) else: try: mod = importlib.import_module(p) new_paths.extend(mod.__path__) forced_local_sys_paths.extend(map(os.path.dirname, mod.__path__)) except ImportError: raise ValueError("custom module {} does not correspond to an existing folder or module".format(p)) paths = new_paths forced_local_sys_paths = sorted(list(set(forced_local_sys_paths))) # collect local sys paths local_sys_paths = copy.copy(sys.path) ## replace empty first element with cwd ## https://docs.python.org/3/library/sys.html#sys.path if local_sys_paths[0] == "": local_sys_paths[0] = os.getcwd() ## convert to absolute paths local_sys_paths = list(map(os.path.abspath, local_sys_paths)) ## remove paths that don't exist local_sys_paths = list(filter(os.path.exists, local_sys_paths)) ## remove .ipython local_sys_paths = list(filter(lambda path: not path.endswith(".ipython"), local_sys_paths)) ## remove virtual (and real) environments local_sys_paths = list(filter(lambda path: not _utils.is_in_venv(path), local_sys_paths)) # get paths to files within if paths is None: # Python files within filtered sys.path dirs paths = local_sys_paths extensions = ['py', 'pyc', 'pyo'] else: # all user-specified files paths = paths extensions = None local_filepaths = _utils.find_filepaths( paths, extensions=extensions, include_hidden=True, include_venv=False, # ignore virtual environments nested within ) ## remove .git local_filepaths = set(filter(lambda path: not path.endswith(".git") and ".git/" not in path, local_filepaths)) # obtain deepest common directory # This directory on the local system will be mirrored in `_CUSTOM_MODULES_DIR` in # deployment. curr_dir = os.path.join(os.getcwd(), "") paths_plus = list(local_filepaths) + [curr_dir] common_prefix = os.path.commonprefix(paths_plus) common_dir = os.path.dirname(common_prefix) # replace `common_dir` with `_CUSTOM_MODULES_DIR` for deployment sys.path depl_sys_paths = list(map(lambda path: os.path.relpath(path, common_dir), local_sys_paths + forced_local_sys_paths)) depl_sys_paths = list(map(lambda path: os.path.join(_CUSTOM_MODULES_DIR, path), depl_sys_paths)) bytestream = six.BytesIO() with zipfile.ZipFile(bytestream, 'w') as zipf: for filepath in local_filepaths: arcname = os.path.relpath(filepath, common_dir) # filepath relative to archive root try: zipf.write(filepath, arcname) except: # maybe file has corrupt metadata; try reading then writing contents with open(filepath, 'rb') as f: zipf.writestr( _artifact_utils.global_read_zipinfo(arcname), f.read(), ) # add verta config file for sys.path and chdir working_dir = os.path.join(_CUSTOM_MODULES_DIR, os.path.relpath(curr_dir, common_dir)) zipf.writestr( _artifact_utils.global_read_zipinfo("_verta_config.py"), six.ensure_binary('\n'.join([ "import os, sys", "", "", "sys.path = sys.path[:1] + {} + sys.path[1:]".format(depl_sys_paths), "", "try:", " os.makedirs(\"{}\")".format(working_dir), "except OSError: # already exists", " pass", "os.chdir(\"{}\")".format(working_dir), ])) ) # add __init__.py init_filename = "__init__.py" if init_filename not in zipf.namelist(): zipf.writestr( _artifact_utils.global_read_zipinfo(init_filename), b"", ) bytestream.seek(0) return bytestream
def get_code(self): """ Gets the code version. Returns ------- dict or zipfile.ZipFile Either: - a dictionary containing Git snapshot information with at most the following items: - **filepaths** (*list of str*) - **repo_url** (*str*) – Remote repository URL - **commit_hash** (*str*) – Commit hash - **is_dirty** (*bool*) - a `ZipFile <https://docs.python.org/3/library/zipfile.html#zipfile.ZipFile>`_ containing Python source code files """ # TODO: remove this circular dependency from ._project import Project from ._experiment import Experiment from ._experimentrun import ExperimentRun if isinstance(self, Project): # TODO: not this Message = self._service.GetProjectCodeVersion endpoint = "getProjectCodeVersion" elif isinstance(self, Experiment): Message = self._service.GetExperimentCodeVersion endpoint = "getExperimentCodeVersion" elif isinstance(self, ExperimentRun): Message = self._service.GetExperimentRunCodeVersion endpoint = "getExperimentRunCodeVersion" msg = Message(id=self.id) data = _utils.proto_to_json(msg) response = _utils.make_request("GET", self._request_url.format(endpoint), self._conn, params=data) _utils.raise_for_http_error(response) response_msg = _utils.json_to_proto(_utils.body_to_json(response), Message.Response) code_ver_msg = response_msg.code_version which_code = code_ver_msg.WhichOneof('code') if which_code == 'git_snapshot': git_snapshot_msg = code_ver_msg.git_snapshot git_snapshot = {} if git_snapshot_msg.filepaths: git_snapshot['filepaths'] = git_snapshot_msg.filepaths if git_snapshot_msg.repo: git_snapshot['repo_url'] = git_snapshot_msg.repo if git_snapshot_msg.hash: git_snapshot['commit_hash'] = git_snapshot_msg.hash if git_snapshot_msg.is_dirty != _CommonCommonService.TernaryEnum.UNKNOWN: git_snapshot[ 'is_dirty'] = git_snapshot_msg.is_dirty == _CommonCommonService.TernaryEnum.TRUE return git_snapshot elif which_code == 'code_archive': # download artifact from artifact store # pylint: disable=no-member # this method should only be called on ExperimentRun, which does have _get_url_for_artifact() url = self._get_url_for_artifact( "verta_code_archive", "GET", code_ver_msg.code_archive.artifact_type).url response = _utils.make_request("GET", url, self._conn) _utils.raise_for_http_error(response) code_archive = six.BytesIO(response.content) return zipfile.ZipFile( code_archive, 'r') # TODO: return a util class instead, maybe else: raise RuntimeError("unable find code in response")
def log_code(self, exec_path=None, repo_url=None, commit_hash=None, overwrite=False, is_dirty=None, autocapture=True): """ Logs the code version. A code version is either information about a Git snapshot or a bundle of Python source code files. `repo_url` and `commit_hash` can only be set if `use_git` was set to ``True`` in the Client. Parameters ---------- exec_path : str, optional Filepath to the executable Python script or Jupyter notebook. If no filepath is provided, the Client will make its best effort to find the currently running script/notebook file. repo_url : str, optional URL for a remote Git repository containing `commit_hash`. If no URL is provided, the Client will make its best effort to find it. commit_hash : str, optional Git commit hash associated with this code version. If no hash is provided, the Client will make its best effort to find it. overwrite : bool, default False Whether to allow overwriting a code version. is_dirty : bool, optional Whether git status is dirty relative to `commit_hash`. If not provided, the Client will make its best effort to find it. autocapture : bool, default True Whether to enable the automatic capturing behavior of parameters above in git mode. Examples -------- With ``Client(use_git=True)`` (default): Log Git snapshot information, plus the location of the currently executing notebook/script relative to the repository root: .. code-block:: python run.log_code() run.get_code() # {'exec_path': 'comparison/outcomes/classification.ipynb', # 'repo_url': '[email protected]:VertaAI/experiments.git', # 'commit_hash': 'f99abcfae6c3ce6d22597f95ad6ef260d31527a6', # 'is_dirty': False} Log Git snapshot information, plus the location of a specific source code file relative to the repository root: .. code-block:: python run.log_code("../trainer/training_pipeline.py") run.get_code() # {'exec_path': 'comparison/trainer/training_pipeline.py', # 'repo_url': '[email protected]:VertaAI/experiments.git', # 'commit_hash': 'f99abcfae6c3ce6d22597f95ad6ef260d31527a6', # 'is_dirty': False} With ``Client(use_git=False)``: Find and upload the currently executing notebook/script: .. code-block:: python run.log_code() zip_file = run.get_code() zip_file.printdir() # File Name Modified Size # classification.ipynb 2019-07-10 17:18:24 10287 Upload a specific source code file: .. code-block:: python run.log_code("../trainer/training_pipeline.py") zip_file = run.get_code() zip_file.printdir() # File Name Modified Size # training_pipeline.py 2019-05-31 10:34:44 964 """ if self._conf.use_git and autocapture: # verify Git try: repo_root_dir = _git_utils.get_git_repo_root_dir() except OSError: # don't halt execution print( "unable to locate git repository; you may be in an unsupported environment;\n" " consider using `autocapture=False` to manually enter values" ) return # six.raise_from(OSError("failed to locate git repository; please check your working directory"), # None) print("Git repository successfully located at {}".format( repo_root_dir)) if not self._conf.use_git: if repo_url is not None or commit_hash is not None: raise ValueError( "`repo_url` and `commit_hash` can only be set if `use_git` was set to True in the Client" ) if not autocapture: # user passed `False` raise ValueError( "`autocapture` is only applicable if `use_git` was set to True in the Client" ) if autocapture: if exec_path is None: # find dynamically try: exec_path = _utils.get_notebook_filepath() except (ImportError, OSError): # notebook not found try: exec_path = _utils.get_script_filepath() except OSError: # script not found print("unable to find code file; skipping") else: exec_path = os.path.expanduser(exec_path) if not os.path.isfile(exec_path): raise ValueError( "`exec_path` \"{}\" must be a valid filepath".format( exec_path)) # TODO: remove this circular dependency from ._project import Project from ._experiment import Experiment from ._experimentrun import ExperimentRun if isinstance(self, Project): # TODO: not this Message = self._service.LogProjectCodeVersion endpoint = "logProjectCodeVersion" elif isinstance(self, Experiment): Message = self._service.LogExperimentCodeVersion endpoint = "logExperimentCodeVersion" elif isinstance(self, ExperimentRun): Message = self._service.LogExperimentRunCodeVersion endpoint = "logExperimentRunCodeVersion" msg = Message(id=self.id) if overwrite: if isinstance(self, ExperimentRun): msg.overwrite = True else: raise ValueError( "`overwrite=True` is currently only supported for ExperimentRun" ) if self._conf.use_git: if autocapture: try: # adjust `exec_path` to be relative to repo root exec_path = os.path.relpath( exec_path, _git_utils.get_git_repo_root_dir()) except OSError as e: print( "{}; logging absolute path to file instead".format(e)) exec_path = os.path.abspath(exec_path) if exec_path: msg.code_version.git_snapshot.filepaths.append(exec_path) from verta.code import _git # avoid Python 2 top-level circular import code_ver = _git.Git( repo_url=repo_url, commit_hash=commit_hash, is_dirty=is_dirty, autocapture=autocapture, ) msg.code_version.git_snapshot.repo = code_ver.repo_url or "" msg.code_version.git_snapshot.hash = code_ver.commit_hash or "" if not autocapture and is_dirty is None: msg.code_version.git_snapshot.is_dirty = _CommonCommonService.TernaryEnum.UNKNOWN elif code_ver.is_dirty: msg.code_version.git_snapshot.is_dirty = _CommonCommonService.TernaryEnum.TRUE else: msg.code_version.git_snapshot.is_dirty = _CommonCommonService.TernaryEnum.FALSE else: # log code as Artifact if exec_path is None: # don't halt execution print( "unable to find code file; you may be in an unsupported environment" ) return # raise RuntimeError("unable to find code file; you may be in an unsupported environment") # write ZIP archive zipstream = six.BytesIO() with zipfile.ZipFile(zipstream, 'w') as zipf: filename = os.path.basename(exec_path) if exec_path.endswith(".ipynb"): try: saved_notebook = _utils.save_notebook(exec_path) except: # failed to save print("unable to automatically save Notebook;" " logging latest checkpoint from disk") zipf.write(exec_path, filename) else: zipf.writestr( _artifact_utils.global_read_zipinfo(filename), six.ensure_binary(saved_notebook.read()), ) else: zipf.write(exec_path, filename) zipstream.seek(0) key = 'code' extension = 'zip' artifact_hash = _artifact_utils.calc_sha256(zipstream) basename = key + os.extsep + extension artifact_path = os.path.join(artifact_hash, basename) msg.code_version.code_archive.path = artifact_path msg.code_version.code_archive.path_only = False msg.code_version.code_archive.artifact_type = _CommonCommonService.ArtifactTypeEnum.CODE msg.code_version.code_archive.filename_extension = extension # TODO: check if we actually have any loggable information msg.code_version.date_logged = _utils.now() data = _utils.proto_to_json(msg) response = _utils.make_request("POST", self._request_url.format(endpoint), self._conn, json=data) if not response.ok: if response.status_code == 409: raise ValueError("a code version has already been logged") else: _utils.raise_for_http_error(response) if msg.code_version.WhichOneof("code") == 'code_archive': # upload artifact to artifact store # pylint: disable=no-member # this method should only be called on ExperimentRun, which does have _get_url_for_artifact() url = self._get_url_for_artifact( "verta_code_archive", "PUT", msg.code_version.code_archive.artifact_type).url response = _utils.make_request("PUT", url, self._conn, data=zipstream) _utils.raise_for_http_error(response)
def _upload_artifact(self, key, file_handle, artifact_type, part_size=_artifact_utils._64MB): file_handle.seek(0) # check if multipart upload ok url_for_artifact = self._get_url_for_artifact(key, "PUT", artifact_type, part_num=1) print("uploading {} to Registry".format(key)) if url_for_artifact.multipart_upload_ok: # TODO: parallelize this file_parts = iter(lambda: file_handle.read(part_size), b"") for part_num, file_part in enumerate(file_parts, start=1): print("uploading part {}".format(part_num), end="\r") # get presigned URL url = self._get_url_for_artifact(key, "PUT", artifact_type, part_num=part_num).url # wrap file part into bytestream to avoid OverflowError # Passing a bytestring >2 GB (num bytes > max val of int32) directly to # ``requests`` will overwhelm CPython's SSL lib when it tries to sign the # payload. But passing a buffered bytestream instead of the raw bytestring # indicates to ``requests`` that it should perform a streaming upload via # HTTP/1.1 chunked transfer encoding and avoid this issue. # https://github.com/psf/requests/issues/2717 part_stream = six.BytesIO(file_part) # upload part response = _utils.make_request("PUT", url, self._conn, data=part_stream) _utils.raise_for_http_error(response) # commit part url = "{}://{}/api/v1/registry/model_versions/{}/commitArtifactPart".format( self._conn.scheme, self._conn.socket, self.id) msg = _RegistryService.CommitArtifactPart( model_version_id=self.id, key=key) msg.artifact_part.part_number = part_num msg.artifact_part.etag = response.headers["ETag"] data = _utils.proto_to_json(msg) response = _utils.make_request("POST", url, self._conn, json=data) _utils.raise_for_http_error(response) print() # complete upload url = "{}://{}/api/v1/registry/model_versions/{}/commitMultipartArtifact".format( self._conn.scheme, self._conn.socket, self.id) msg = _RegistryService.CommitMultipartArtifact( model_version_id=self.id, key=key) data = _utils.proto_to_json(msg) response = _utils.make_request("POST", url, self._conn, json=data) _utils.raise_for_http_error(response) else: # upload full artifact if url_for_artifact.fields: # if fields were returned by backend, make a POST request and supply them as form fields response = _utils.make_request( "POST", url_for_artifact.url, self._conn, # requests uses the `files` parameter for sending multipart/form-data POSTs. # https://stackoverflow.com/a/12385661/8651995 # the file contents must be the final form field # https://docs.aws.amazon.com/AmazonS3/latest/dev/HTTPPOSTForms.html#HTTPPOSTFormFields files=list(url_for_artifact.fields.items()) + [("file", file_handle)], ) else: response = _utils.make_request("PUT", url_for_artifact.url, self._conn, data=file_handle) _utils.raise_for_http_error(response) print("upload complete")
def _upload_artifact(self, dataset_component_path, file_handle, part_size=_artifact_utils._64MB): """ Uploads `file_handle` to ModelDB artifact store. Parameters ---------- dataset_component_path : str Filepath in dataset component blob. file_handle : file-like Artifact to be uploaded. part_size : int, default 64 MB If using multipart upload, number of bytes to upload per part. """ file_handle.seek(0) # check if multipart upload ok url_for_artifact = self._get_url_for_artifact(dataset_component_path, "PUT", part_num=1) print("uploading {} to ModelDB".format(dataset_component_path)) if url_for_artifact.multipart_upload_ok: # TODO: parallelize this file_parts = iter(lambda: file_handle.read(part_size), b'') for part_num, file_part in enumerate(file_parts, start=1): print("uploading part {}".format(part_num), end='\r') # get presigned URL url = self._get_url_for_artifact(dataset_component_path, "PUT", part_num=part_num).url # wrap file part into bytestream to avoid OverflowError # Passing a bytestring >2 GB (num bytes > max val of int32) directly to # ``requests`` will overwhelm CPython's SSL lib when it tries to sign the # payload. But passing a buffered bytestream instead of the raw bytestring # indicates to ``requests`` that it should perform a streaming upload via # HTTP/1.1 chunked transfer encoding and avoid this issue. # https://github.com/psf/requests/issues/2717 part_stream = six.BytesIO(file_part) # upload part response = _utils.make_request("PUT", url, self._conn, data=part_stream) _utils.raise_for_http_error(response) # commit part url = "{}://{}/api/v1/modeldb/dataset-version/commitVersionedDatasetBlobArtifactPart".format( self._conn.scheme, self._conn.socket, ) msg = _DatasetVersionService.CommitVersionedDatasetBlobArtifactPart( dataset_version_id=self.id, path_dataset_component_blob_path=dataset_component_path, ) msg.artifact_part.part_number = part_num msg.artifact_part.etag = response.headers['ETag'] data = _utils.proto_to_json(msg) response = _utils.make_request("POST", url, self._conn, json=data) _utils.raise_for_http_error(response) print() # complete upload url = "{}://{}/api/v1/modeldb/dataset-version/commitMultipartVersionedDatasetBlobArtifact".format( self._conn.scheme, self._conn.socket, ) msg = _DatasetVersionService.CommitMultipartVersionedDatasetBlobArtifact( dataset_version_id=self.id, path_dataset_component_blob_path=dataset_component_path, ) data = _utils.proto_to_json(msg) response = _utils.make_request("POST", url, self._conn, json=data) _utils.raise_for_http_error(response) else: # upload full artifact if url_for_artifact.fields: # if fields were returned by backend, make a POST request and supply them as form fields response = _utils.make_request( "POST", url_for_artifact.url, self._conn, # requests uses the `files` parameter for sending multipart/form-data POSTs. # https://stackoverflow.com/a/12385661/8651995 # the file contents must be the final form field # https://docs.aws.amazon.com/AmazonS3/latest/dev/HTTPPOSTForms.html#HTTPPOSTFormFields files=list(url_for_artifact.fields.items()) + [('file', file_handle)], ) else: response = _utils.make_request("PUT", url_for_artifact.url, self._conn, data=file_handle) _utils.raise_for_http_error(response) print("upload complete")