def _get_run_files(self, run_uuid, resource_type):
_validate_run_id(run_uuid)
run_info = self._get_run_info(run_uuid)
if run_info is None:
raise MlflowException(
"Run '%s' metadata is in invalid state." % run_uuid,
databricks_pb2.INVALID_STATE)
if resource_type == "metric":
subfolder_name = FileStore.METRICS_FOLDER_NAME
elif resource_type == "param":
subfolder_name = FileStore.PARAMS_FOLDER_NAME
elif resource_type == "tag":
subfolder_name = FileStore.TAGS_FOLDER_NAME
else:
raise Exception("Looking for unknown resource under run.")
_, run_dir = self._find_run_root(run_uuid)
# run_dir exists since run validity has been confirmed above.
source_dirs = find(run_dir, subfolder_name, full_path=True)
if len(source_dirs) == 0:
return run_dir, []
file_names = []
for root, _, files in os.walk(source_dirs[0]):
for name in files:
abspath = os.path.join(root, name)
file_names.append(os.path.relpath(abspath, source_dirs[0]))
if sys.platform == "win32":
# Turn metric relative path into metric name.
# Metrics can have '/' in the name. On windows, '/' is interpreted as a separator.
# When the metric is read back the path will use '\' for separator.
# We need to translate the path into posix path.
from mlflow.utils.file_utils import relative_path_to_artifact_path
file_names = [
relative_path_to_artifact_path(x) for x in file_names
]
return source_dirs[0], file_names
def _get_run_files(self, run_uuid, resource_type):
_validate_run_id(run_uuid)
run_info = self._get_run_info(run_uuid)
if run_info is None:
raise MlflowException(
"Run '%s' metadata is in invalid state." % run_uuid,
databricks_pb2.INVALID_STATE)
if resource_type == "metric":
subfolder_name = FileStore.METRICS_FOLDER_NAME
elif resource_type == "param":
subfolder_name = FileStore.PARAMS_FOLDER_NAME
elif resource_type == "tag":
subfolder_name = FileStore.TAGS_FOLDER_NAME
else:
raise Exception("Looking for unknown resource under run.")
_, run_dir = self._find_run_root(run_uuid)
# run_dir exists since run validity has been confirmed above.
source_dirs = find(run_dir, subfolder_name, full_path=True)
if len(source_dirs) == 0:
return run_dir, []
file_names = []
for root, _, files in os.walk(source_dirs[0]):
for name in files:
abspath = os.path.join(root, name)
file_names.append(os.path.relpath(abspath, source_dirs[0]))
return source_dirs[0], file_names
def _find_run_root(self, run_uuid):
self._check_root_dir()
all_experiments = list_subdirs(self.root_directory, full_path=True)
for experiment_dir in all_experiments:
runs = find(experiment_dir, run_uuid, full_path=True)
if len(runs) == 0:
continue
return runs[0]
return None
def get_experiment(self, experiment_id):
self._check_root_dir()
experiment_dirs = find(self.root_directory,
str(experiment_id),
full_path=True)
if len(experiment_dirs) == 0:
raise Exception("Could not find experiment with ID %s" %
experiment_id)
return self._get_experiment(experiment_dirs[0])
def _find_run_root(self, run_uuid):
_validate_run_id(run_uuid)
self._check_root_dir()
all_experiments = self._get_active_experiments(True) + self._get_deleted_experiments(True)
for experiment_dir in all_experiments:
runs = find(experiment_dir, run_uuid, full_path=True)
if len(runs) == 0:
continue
return os.path.basename(os.path.abspath(experiment_dir)), runs[0]
return None, None
def _get_experiment_path(self, experiment_id, view_type=ViewType.ALL):
parents = []
if view_type == ViewType.ACTIVE_ONLY or view_type == ViewType.ALL:
parents.append(self.root_directory)
if view_type == ViewType.DELETED_ONLY or view_type == ViewType.ALL:
parents.append(self.trash_folder)
for parent in parents:
exp_list = find(parent, str(experiment_id), full_path=True)
if len(exp_list) > 0:
return exp_list
return []
def _get_run_files(self, run_uuid, resource_type):
if resource_type == "metric":
subfolder_name = FileStore.METRICS_FOLDER_NAME
elif resource_type == "param":
subfolder_name = FileStore.PARAMS_FOLDER_NAME
else:
raise Exception("Looking for unknown resource under run.")
run_dir = self._find_run_root(run_uuid)
if run_dir is None:
raise Exception("Run '%s' not found" % run_uuid)
source_dirs = find(run_dir, subfolder_name, full_path=True)
if len(source_dirs) == 0:
raise Exception("Malformed run '%s'." % run_uuid)
return source_dirs[0], list_files(source_dirs[0], full_path=False)
def _get_experiment_path(self, experiment_id, view_type=ViewType.ALL, assert_exists=False):
parents = []
if view_type == ViewType.ACTIVE_ONLY or view_type == ViewType.ALL:
parents.append(self.root_directory)
if view_type == ViewType.DELETED_ONLY or view_type == ViewType.ALL:
parents.append(self.trash_folder)
for parent in parents:
exp_list = find(parent, experiment_id, full_path=True)
if len(exp_list) > 0:
return exp_list[0]
if assert_exists:
raise MlflowException('Experiment {} does not exist.'.format(experiment_id),
databricks_pb2.RESOURCE_DOES_NOT_EXIST)
return None
def _get_resource_files(self, root_dir, subfolder_name):
source_dirs = find(root_dir, subfolder_name, full_path=True)
if len(source_dirs) == 0:
return root_dir, []
file_names = []
for root, _, files in os.walk(source_dirs[0]):
for name in files:
abspath = os.path.join(root, name)
file_names.append(os.path.relpath(abspath, source_dirs[0]))
if sys.platform == "win32":
# Turn metric relative path into metric name.
# Metrics can have '/' in the name. On windows, '/' is interpreted as a separator.
# When the metric is read back the path will use '\' for separator.
# We need to translate the path into posix path.
from mlflow.utils.file_utils import relative_path_to_artifact_path
file_names = [relative_path_to_artifact_path(x) for x in file_names]
return source_dirs[0], file_names
def _get_run_files(self, run_uuid, resource_type):
if resource_type == "metric":
subfolder_name = FileStore.METRICS_FOLDER_NAME
elif resource_type == "param":
subfolder_name = FileStore.PARAMS_FOLDER_NAME
else:
raise Exception("Looking for unknown resource under run.")
run_dir = self._find_run_root(run_uuid)
if run_dir is None:
raise Exception("Run '%s' not found" % run_uuid)
source_dirs = find(run_dir, subfolder_name, full_path=True)
if len(source_dirs) == 0:
raise Exception("Malformed run '%s'." % run_uuid)
file_names = []
for root, _, files in os.walk(source_dirs[0]):
for name in files:
abspath = os.path.join(root, name)
file_names.append(os.path.relpath(abspath, source_dirs[0]))
return source_dirs[0], file_names
def _get_run_files(self, run_uuid, resource_type):
_validate_run_id(run_uuid)
if resource_type == "metric":
subfolder_name = FileStore.METRICS_FOLDER_NAME
elif resource_type == "param":
subfolder_name = FileStore.PARAMS_FOLDER_NAME
elif resource_type == "tag":
subfolder_name = FileStore.TAGS_FOLDER_NAME
else:
raise Exception("Looking for unknown resource under run.")
run_dir = self._find_run_root(run_uuid)
if run_dir is None:
raise MlflowException("Run '%s' not found" % run_uuid,
databricks_pb2.RESOURCE_DOES_NOT_EXIST)
source_dirs = find(run_dir, subfolder_name, full_path=True)
if len(source_dirs) == 0:
return run_dir, []
file_names = []
for root, _, files in os.walk(source_dirs[0]):
for name in files:
abspath = os.path.join(root, name)
file_names.append(os.path.relpath(abspath, source_dirs[0]))
return source_dirs[0], file_names
def _list_run_uuids(self, experiment_id):
self._check_root_dir()
experiment_dir = find(self.root_directory,
str(experiment_id),
full_path=True)[0]
return list_subdirs(experiment_dir, full_path=False)
def _has_experiment(self, experiment_id):
return len(
find(self.root_directory, str(experiment_id), full_path=True)) > 0
