def log_artifacts(self, local_dir, artifact_path=None):
if artifact_path:
root_http_endpoint = self._get_dbfs_endpoint(artifact_path)
else:
root_http_endpoint = self._get_dbfs_endpoint(
os.path.basename(local_dir))
for (dirpath, _, filenames) in os.walk(local_dir):
dir_http_endpoint = root_http_endpoint
if dirpath != local_dir:
rel_path = get_relative_path(local_dir, dirpath)
dir_http_endpoint = build_path(root_http_endpoint, rel_path)
for name in filenames:
endpoint = build_path(dir_http_endpoint, name)
with open(build_path(dirpath, name), 'rb') as f:
response = http_request(endpoint=endpoint,
method='POST',
data=f,
allow_redirects=False,
**self.http_request_kwargs)
if response.status_code == 409:
raise MlflowException(
'File already exists at {} and can\'t be overwritten.'.
format(endpoint))
elif response.status_code != 200:
raise MlflowException(
'log_artifacts to "{}" returned a non-200 status code.'
.format(endpoint))
def list_artifacts(self, path=None):
artifact_dir = self.artifact_uri
list_dir = build_path(artifact_dir, path) if path else artifact_dir
artifact_files = list_all(list_dir, full_path=True)
return [
get_file_info(f, get_relative_path(artifact_dir, f))
for f in artifact_files
]
def list_artifacts(self, path=None):
artifact_dir = self.artifact_uri
list_dir = build_path(artifact_dir, path) if path else artifact_dir
artifact_files = list_all(list_dir, full_path=True)
infos = [
get_file_info(f, get_relative_path(artifact_dir, f))
for f in artifact_files
]
return sorted(infos, key=lambda f: f.path)
def log_artifacts(self, local_dir, artifact_path=None):
(bucket, dest_path) = self.parse_s3_uri(self.artifact_uri)
if artifact_path:
dest_path = build_path(dest_path, artifact_path)
s3 = boto3.client('s3')
local_dir = os.path.abspath(local_dir)
for (root, _, filenames) in os.walk(local_dir):
upload_path = dest_path
if root != local_dir:
rel_path = get_relative_path(local_dir, root)
upload_path = build_path(dest_path, rel_path)
for f in filenames:
s3.upload_file(build_path(root, f), bucket, build_path(upload_path, f))
def log_artifacts(self, local_dir, artifact_path=None):
(container, _, dest_path) = self.parse_wasbs_uri(self.artifact_uri)
if artifact_path:
dest_path = build_path(dest_path, artifact_path)
local_dir = os.path.abspath(local_dir)
for (root, _, filenames) in os.walk(local_dir):
upload_path = dest_path
if root != local_dir:
rel_path = get_relative_path(local_dir, root)
upload_path = build_path(dest_path, rel_path)
for f in filenames:
path = build_path(upload_path, f)
self.client.create_blob_from_path(container, path, build_path(root, f))
def log_artifacts(self, local_dir, artifact_path=None):
(bucket, dest_path) = self.parse_gcs_uri(self.artifact_uri)
if artifact_path:
dest_path = build_path(dest_path, artifact_path)
gcs_bucket = self.gcs.Client().get_bucket(bucket)
local_dir = os.path.abspath(local_dir)
for (root, _, filenames) in os.walk(local_dir):
upload_path = dest_path
if root != local_dir:
rel_path = get_relative_path(local_dir, root)
upload_path = build_path(dest_path, rel_path)
for f in filenames:
path = build_path(upload_path, f)
gcs_bucket.blob(path).upload_from_filename(build_path(root, f))
def log_artifacts(self, local_dir, artifact_path=None):
(bucket, dest_path) = data.parse_s3_uri(self.artifact_uri)
if artifact_path:
dest_path = build_path(dest_path, artifact_path)
s3_endpoint_url = os.environ.get('MLFLOW_S3_ENDPOINT_URL')
s3 = boto3.client('s3', endpoint_url=s3_endpoint_url)
local_dir = os.path.abspath(local_dir)
for (root, _, filenames) in os.walk(local_dir):
upload_path = dest_path
if root != local_dir:
rel_path = get_relative_path(local_dir, root)
upload_path = build_path(dest_path, rel_path)
for f in filenames:
s3.upload_file(build_path(root, f), bucket,
build_path(upload_path, f))
def log_artifacts(self, local_dir, artifact_path=None):
if artifact_path:
root_http_endpoint = self._get_dbfs_endpoint(artifact_path)
else:
root_http_endpoint = self._get_dbfs_endpoint(os.path.basename(local_dir))
for (dirpath, _, filenames) in os.walk(local_dir):
dir_http_endpoint = root_http_endpoint
if dirpath != local_dir:
rel_path = get_relative_path(local_dir, dirpath)
dir_http_endpoint = build_path(root_http_endpoint, rel_path)
for name in filenames:
endpoint = build_path(dir_http_endpoint, name)
with open(build_path(dirpath, name), 'r') as f:
http_request(endpoint=endpoint, method='POST', data=f,
**self.http_request_kwargs)
def log_artifacts(self, local_dir, artifact_path=None):
if artifact_path:
root_http_endpoint = self._get_dbfs_endpoint(artifact_path)
else:
root_http_endpoint = self._get_dbfs_endpoint('')
for (dirpath, _, filenames) in os.walk(local_dir):
dir_http_endpoint = root_http_endpoint
if dirpath != local_dir:
rel_path = get_relative_path(local_dir, dirpath)
dir_http_endpoint = build_path(root_http_endpoint, rel_path)
for name in filenames:
endpoint = build_path(dir_http_endpoint, name)
with open(build_path(dirpath, name), 'rb') as f:
self._databricks_api_request(
endpoint=endpoint, method='POST', data=f, allow_redirects=False)
def log_artifacts(self, local_dir, artifact_path=None):
dest_path = os.path.join(self.path, artifact_path) \
if artifact_path else self.path
dest_path = build_path(dest_path, os.path.split(local_dir)[1])
dest_path_re = os.path.split(local_dir)[1]
if artifact_path:
dest_path_re = build_path(artifact_path,
os.path.split(local_dir)[1])
local_dir = os.path.abspath(local_dir)
for (root, _, filenames) in os.walk(local_dir):
upload_path = dest_path
if root != local_dir:
rel_path = get_relative_path(local_dir, root)
upload_path = build_path(dest_path_re, rel_path)
if not filenames:
self._mkdir(build_path(self.path, upload_path))
for f in filenames:
if os.path.isfile(build_path(root, f)):
self.log_artifact(build_path(root, f), upload_path)
