def log_artifacts(self, local_dir, artifact_path=None): if artifact_path: root_http_endpoint = self._get_dbfs_endpoint(artifact_path) else: root_http_endpoint = self._get_dbfs_endpoint( os.path.basename(local_dir)) for (dirpath, _, filenames) in os.walk(local_dir): dir_http_endpoint = root_http_endpoint if dirpath != local_dir: rel_path = get_relative_path(local_dir, dirpath) dir_http_endpoint = build_path(root_http_endpoint, rel_path) for name in filenames: endpoint = build_path(dir_http_endpoint, name) with open(build_path(dirpath, name), 'rb') as f: response = http_request(endpoint=endpoint, method='POST', data=f, allow_redirects=False, **self.http_request_kwargs) if response.status_code == 409: raise MlflowException( 'File already exists at {} and can\'t be overwritten.'. format(endpoint)) elif response.status_code != 200: raise MlflowException( 'log_artifacts to "{}" returned a non-200 status code.' .format(endpoint))
def list_artifacts(self, path=None): artifact_dir = self.artifact_uri list_dir = build_path(artifact_dir, path) if path else artifact_dir artifact_files = list_all(list_dir, full_path=True) return [ get_file_info(f, get_relative_path(artifact_dir, f)) for f in artifact_files ]
def list_artifacts(self, path=None): artifact_dir = self.artifact_uri list_dir = build_path(artifact_dir, path) if path else artifact_dir artifact_files = list_all(list_dir, full_path=True) infos = [ get_file_info(f, get_relative_path(artifact_dir, f)) for f in artifact_files ] return sorted(infos, key=lambda f: f.path)
def log_artifacts(self, local_dir, artifact_path=None): (bucket, dest_path) = self.parse_s3_uri(self.artifact_uri) if artifact_path: dest_path = build_path(dest_path, artifact_path) s3 = boto3.client('s3') local_dir = os.path.abspath(local_dir) for (root, _, filenames) in os.walk(local_dir): upload_path = dest_path if root != local_dir: rel_path = get_relative_path(local_dir, root) upload_path = build_path(dest_path, rel_path) for f in filenames: s3.upload_file(build_path(root, f), bucket, build_path(upload_path, f))
def log_artifacts(self, local_dir, artifact_path=None): (container, _, dest_path) = self.parse_wasbs_uri(self.artifact_uri) if artifact_path: dest_path = build_path(dest_path, artifact_path) local_dir = os.path.abspath(local_dir) for (root, _, filenames) in os.walk(local_dir): upload_path = dest_path if root != local_dir: rel_path = get_relative_path(local_dir, root) upload_path = build_path(dest_path, rel_path) for f in filenames: path = build_path(upload_path, f) self.client.create_blob_from_path(container, path, build_path(root, f))
def log_artifacts(self, local_dir, artifact_path=None): (bucket, dest_path) = self.parse_gcs_uri(self.artifact_uri) if artifact_path: dest_path = build_path(dest_path, artifact_path) gcs_bucket = self.gcs.Client().get_bucket(bucket) local_dir = os.path.abspath(local_dir) for (root, _, filenames) in os.walk(local_dir): upload_path = dest_path if root != local_dir: rel_path = get_relative_path(local_dir, root) upload_path = build_path(dest_path, rel_path) for f in filenames: path = build_path(upload_path, f) gcs_bucket.blob(path).upload_from_filename(build_path(root, f))
def log_artifacts(self, local_dir, artifact_path=None): (bucket, dest_path) = data.parse_s3_uri(self.artifact_uri) if artifact_path: dest_path = build_path(dest_path, artifact_path) s3_endpoint_url = os.environ.get('MLFLOW_S3_ENDPOINT_URL') s3 = boto3.client('s3', endpoint_url=s3_endpoint_url) local_dir = os.path.abspath(local_dir) for (root, _, filenames) in os.walk(local_dir): upload_path = dest_path if root != local_dir: rel_path = get_relative_path(local_dir, root) upload_path = build_path(dest_path, rel_path) for f in filenames: s3.upload_file(build_path(root, f), bucket, build_path(upload_path, f))
def log_artifacts(self, local_dir, artifact_path=None): if artifact_path: root_http_endpoint = self._get_dbfs_endpoint(artifact_path) else: root_http_endpoint = self._get_dbfs_endpoint(os.path.basename(local_dir)) for (dirpath, _, filenames) in os.walk(local_dir): dir_http_endpoint = root_http_endpoint if dirpath != local_dir: rel_path = get_relative_path(local_dir, dirpath) dir_http_endpoint = build_path(root_http_endpoint, rel_path) for name in filenames: endpoint = build_path(dir_http_endpoint, name) with open(build_path(dirpath, name), 'r') as f: http_request(endpoint=endpoint, method='POST', data=f, **self.http_request_kwargs)
def log_artifacts(self, local_dir, artifact_path=None): if artifact_path: root_http_endpoint = self._get_dbfs_endpoint(artifact_path) else: root_http_endpoint = self._get_dbfs_endpoint('') for (dirpath, _, filenames) in os.walk(local_dir): dir_http_endpoint = root_http_endpoint if dirpath != local_dir: rel_path = get_relative_path(local_dir, dirpath) dir_http_endpoint = build_path(root_http_endpoint, rel_path) for name in filenames: endpoint = build_path(dir_http_endpoint, name) with open(build_path(dirpath, name), 'rb') as f: self._databricks_api_request( endpoint=endpoint, method='POST', data=f, allow_redirects=False)
def log_artifacts(self, local_dir, artifact_path=None): dest_path = os.path.join(self.path, artifact_path) \ if artifact_path else self.path dest_path = build_path(dest_path, os.path.split(local_dir)[1]) dest_path_re = os.path.split(local_dir)[1] if artifact_path: dest_path_re = build_path(artifact_path, os.path.split(local_dir)[1]) local_dir = os.path.abspath(local_dir) for (root, _, filenames) in os.walk(local_dir): upload_path = dest_path if root != local_dir: rel_path = get_relative_path(local_dir, root) upload_path = build_path(dest_path_re, rel_path) if not filenames: self._mkdir(build_path(self.path, upload_path)) for f in filenames: if os.path.isfile(build_path(root, f)): self.log_artifact(build_path(root, f), upload_path)