def log_artifacts(self, local_dir, artifact_path=None):
artifact_path = artifact_path or ''
for (dirpath, _, filenames) in os.walk(local_dir):
artifact_subdir = artifact_path
if dirpath != local_dir:
rel_path = os.path.relpath(dirpath, local_dir)
rel_path = relative_path_to_artifact_path(rel_path)
artifact_subdir = posixpath.join(artifact_path, rel_path)
for name in filenames:
file_path = os.path.join(dirpath, name)
self.log_artifact(file_path, artifact_subdir)
def log_artifacts(self, local_dir, artifact_path=None):
(bucket, dest_path) = self.parse_gcs_uri(self.artifact_uri)
if artifact_path:
dest_path = posixpath.join(dest_path, artifact_path)
gcs_bucket = self._get_bucket(bucket)
local_dir = os.path.abspath(local_dir)
for (root, _, filenames) in os.walk(local_dir):
upload_path = dest_path
if root != local_dir:
rel_path = os.path.relpath(root, local_dir)
rel_path = relative_path_to_artifact_path(rel_path)
upload_path = posixpath.join(dest_path, rel_path)
for f in filenames:
path = posixpath.join(upload_path, f)
gcs_bucket.blob(path).upload_from_filename(
os.path.join(root, f))
def log_artifacts(self, local_dir, artifact_path=None):
(bucket, dest_path) = data.parse_s3_uri(self.artifact_uri)
if artifact_path:
dest_path = posixpath.join(dest_path, artifact_path)
s3_client = self._get_s3_client()
local_dir = os.path.abspath(local_dir)
for (root, _, filenames) in os.walk(local_dir):
upload_path = dest_path
if root != local_dir:
rel_path = os.path.relpath(root, local_dir)
rel_path = relative_path_to_artifact_path(rel_path)
upload_path = posixpath.join(dest_path, rel_path)
for f in filenames:
self._upload_file(s3_client=s3_client,
local_file=os.path.join(root, f),
bucket=bucket,
key=posixpath.join(upload_path, f))
def _get_resource_files(self, root_dir, subfolder_name):
source_dirs = find(root_dir, subfolder_name, full_path=True)
if len(source_dirs) == 0:
return root_dir, []
file_names = []
for root, _, files in os.walk(source_dirs[0]):
for name in files:
abspath = os.path.join(root, name)
file_names.append(os.path.relpath(abspath, source_dirs[0]))
if sys.platform == "win32":
# Turn metric relative path into metric name.
# Metrics can have '/' in the name. On windows, '/' is interpreted as a separator.
# When the metric is read back the path will use '\' for separator.
# We need to translate the path into posix path.
from kiwi.utils.file_utils import relative_path_to_artifact_path
file_names = [
relative_path_to_artifact_path(x) for x in file_names
]
return source_dirs[0], file_names
def list_artifacts(self, path=None):
# NOTE: The path is expected to be in posix format.
# Posix paths work fine on windows but just in case we normalize it here.
if path:
path = os.path.normpath(path)
list_dir = os.path.join(self.artifact_dir,
path) if path else self.artifact_dir
if os.path.isdir(list_dir):
artifact_files = list_all(list_dir, full_path=True)
infos = [
get_file_info(
f,
relative_path_to_artifact_path(
os.path.relpath(f, self.artifact_dir)))
for f in artifact_files
]
return sorted(infos, key=lambda f: f.path)
else:
return []
def log_artifacts(self, local_dir, artifact_path=None):
dest_path = posixpath.join(self.path, artifact_path) \
if artifact_path else self.path
dest_path = posixpath.join(dest_path, os.path.split(local_dir)[1])
dest_path_re = os.path.split(local_dir)[1]
if artifact_path:
dest_path_re = posixpath.join(artifact_path,
os.path.split(local_dir)[1])
local_dir = os.path.abspath(local_dir)
for (root, _, filenames) in os.walk(local_dir):
upload_path = dest_path
if root != local_dir:
rel_path = os.path.relpath(root, local_dir)
rel_path = relative_path_to_artifact_path(rel_path)
upload_path = posixpath.join(dest_path_re, rel_path)
if not filenames:
with self.get_ftp_client() as ftp:
self._mkdir(ftp, posixpath.join(self.path, upload_path))
for f in filenames:
if os.path.isfile(os.path.join(root, f)):
self.log_artifact(os.path.join(root, f), upload_path)
def _relative_path_local(base_dir, subdir_path):
rel_path = _relative_path(base_dir, subdir_path, os.path)
return relative_path_to_artifact_path(rel_path) if rel_path is not None else None