def log_artifacts(self, local_dir, artifact_path=None): artifact_path = artifact_path or '' for (dirpath, _, filenames) in os.walk(local_dir): artifact_subdir = artifact_path if dirpath != local_dir: rel_path = os.path.relpath(dirpath, local_dir) rel_path = relative_path_to_artifact_path(rel_path) artifact_subdir = posixpath.join(artifact_path, rel_path) for name in filenames: file_path = os.path.join(dirpath, name) self.log_artifact(file_path, artifact_subdir)
def log_artifacts(self, local_dir, artifact_path=None): (bucket, dest_path) = self.parse_gcs_uri(self.artifact_uri) if artifact_path: dest_path = posixpath.join(dest_path, artifact_path) gcs_bucket = self._get_bucket(bucket) local_dir = os.path.abspath(local_dir) for (root, _, filenames) in os.walk(local_dir): upload_path = dest_path if root != local_dir: rel_path = os.path.relpath(root, local_dir) rel_path = relative_path_to_artifact_path(rel_path) upload_path = posixpath.join(dest_path, rel_path) for f in filenames: path = posixpath.join(upload_path, f) gcs_bucket.blob(path).upload_from_filename( os.path.join(root, f))
def log_artifacts(self, local_dir, artifact_path=None): (bucket, dest_path) = data.parse_s3_uri(self.artifact_uri) if artifact_path: dest_path = posixpath.join(dest_path, artifact_path) s3_client = self._get_s3_client() local_dir = os.path.abspath(local_dir) for (root, _, filenames) in os.walk(local_dir): upload_path = dest_path if root != local_dir: rel_path = os.path.relpath(root, local_dir) rel_path = relative_path_to_artifact_path(rel_path) upload_path = posixpath.join(dest_path, rel_path) for f in filenames: self._upload_file(s3_client=s3_client, local_file=os.path.join(root, f), bucket=bucket, key=posixpath.join(upload_path, f))
def _get_resource_files(self, root_dir, subfolder_name): source_dirs = find(root_dir, subfolder_name, full_path=True) if len(source_dirs) == 0: return root_dir, [] file_names = [] for root, _, files in os.walk(source_dirs[0]): for name in files: abspath = os.path.join(root, name) file_names.append(os.path.relpath(abspath, source_dirs[0])) if sys.platform == "win32": # Turn metric relative path into metric name. # Metrics can have '/' in the name. On windows, '/' is interpreted as a separator. # When the metric is read back the path will use '\' for separator. # We need to translate the path into posix path. from kiwi.utils.file_utils import relative_path_to_artifact_path file_names = [ relative_path_to_artifact_path(x) for x in file_names ] return source_dirs[0], file_names
def list_artifacts(self, path=None): # NOTE: The path is expected to be in posix format. # Posix paths work fine on windows but just in case we normalize it here. if path: path = os.path.normpath(path) list_dir = os.path.join(self.artifact_dir, path) if path else self.artifact_dir if os.path.isdir(list_dir): artifact_files = list_all(list_dir, full_path=True) infos = [ get_file_info( f, relative_path_to_artifact_path( os.path.relpath(f, self.artifact_dir))) for f in artifact_files ] return sorted(infos, key=lambda f: f.path) else: return []
def log_artifacts(self, local_dir, artifact_path=None): dest_path = posixpath.join(self.path, artifact_path) \ if artifact_path else self.path dest_path = posixpath.join(dest_path, os.path.split(local_dir)[1]) dest_path_re = os.path.split(local_dir)[1] if artifact_path: dest_path_re = posixpath.join(artifact_path, os.path.split(local_dir)[1]) local_dir = os.path.abspath(local_dir) for (root, _, filenames) in os.walk(local_dir): upload_path = dest_path if root != local_dir: rel_path = os.path.relpath(root, local_dir) rel_path = relative_path_to_artifact_path(rel_path) upload_path = posixpath.join(dest_path_re, rel_path) if not filenames: with self.get_ftp_client() as ftp: self._mkdir(ftp, posixpath.join(self.path, upload_path)) for f in filenames: if os.path.isfile(os.path.join(root, f)): self.log_artifact(os.path.join(root, f), upload_path)
def _relative_path_local(base_dir, subdir_path): rel_path = _relative_path(base_dir, subdir_path, os.path) return relative_path_to_artifact_path(rel_path) if rel_path is not None else None