def _create_experiment_dir(app_id, run_id):
experiment_path = _get_logdir(app_id, run_id)
if hdfs.exists(experiment_path):
hdfs.delete(experiment_path, recursive=True)
hdfs.mkdir(experiment_path)
def _clean_dir(clean_dir, keep=[]):
"""Deletes all files in a directory but keeps a few.
"""
if not hopshdfs.isdir(clean_dir):
raise ValueError(
"{} is not a directory. Use `hops.hdfs.delete()` to delete single "
"files.".format(clean_dir))
for path in hopshdfs.ls(clean_dir):
if path not in keep:
hopshdfs.delete(path, recursive=True)
def _upload_file_output(retval, hdfs_exec_logdir):
if type(retval) is dict:
for metric_key in retval.keys():
value = str(retval[metric_key])
if '/' in value or os.path.exists(os.getcwd() + '/' + value):
if os.path.exists(value): # absolute path
if hdfs.exists(hdfs_exec_logdir + '/' +
value.split('/')[-1]):
hdfs.delete(hdfs_exec_logdir + '/' +
value.split('/')[-1],
recursive=False)
pydoop.hdfs.put(value, hdfs_exec_logdir)
os.remove(value)
hdfs_exec_logdir = hdfs.abs_path(hdfs_exec_logdir)
retval[metric_key] = hdfs_exec_logdir[
len(hdfs.abs_path(hdfs.project_path())
):] + '/' + value.split('/')[-1]
elif os.path.exists(os.getcwd() + '/' +
value): # relative path
output_file = os.getcwd() + '/' + value
if hdfs.exists(hdfs_exec_logdir + '/' + value):
hdfs.delete(hdfs_exec_logdir + '/' + value,
recursive=False)
pydoop.hdfs.put(value, hdfs_exec_logdir)
os.remove(output_file)
hdfs_exec_logdir = hdfs.abs_path(hdfs_exec_logdir)
retval[metric_key] = hdfs_exec_logdir[
len(hdfs.abs_path(hdfs.project_path())
):] + '/' + output_file.split('/')[-1]
elif value.startswith('Experiments') and value.endswith(
'output.log'):
continue
elif value.startswith('Experiments') and hdfs.exists(
hdfs.project_path() + '/' + value):
hdfs.cp(hdfs.project_path() + '/' + value,
hdfs_exec_logdir)
else:
raise Exception(
'Could not find file or directory on path ' +
str(value))
def _create_experiment_subdirectories(app_id,
run_id,
param_string,
type,
sub_type=None,
params=None):
"""
Creates directories for an experiment, if Experiments folder exists it will create directories
below it, otherwise it will create them in the Logs directory.
Args:
:app_id: YARN application ID of the experiment
:run_id: Experiment ID
:param_string: name of the new directory created under parent directories
:type: type of the new directory parent, e.g differential_evolution
:sub_type: type of sub directory to parent, e.g generation
:params: dict of hyperparameters
Returns:
The new directories for the yarn-application and for the execution (hdfs_exec_logdir, hdfs_appid_logdir)
"""
pyhdfs_handle = hdfs.get()
hdfs_events_parent_dir = hdfs.project_path() + "Experiments"
hdfs_experiment_dir = hdfs_events_parent_dir + "/" + app_id + "_" + str(
run_id)
# determine directory structure based on arguments
if sub_type:
hdfs_exec_logdir = hdfs_experiment_dir + "/" + str(
sub_type) + '/' + str(param_string)
if pyhdfs_handle.exists(hdfs_exec_logdir):
hdfs.delete(hdfs_exec_logdir, recursive=True)
elif not param_string and not sub_type:
if pyhdfs_handle.exists(hdfs_experiment_dir):
hdfs.delete(hdfs_experiment_dir, recursive=True)
hdfs_exec_logdir = hdfs_experiment_dir + '/'
else:
hdfs_exec_logdir = hdfs_experiment_dir + '/' + str(param_string)
if pyhdfs_handle.exists(hdfs_exec_logdir):
hdfs.delete(hdfs_exec_logdir, recursive=True)
# Need to remove directory if it exists (might be a task retry)
# create the new directory
pyhdfs_handle.create_directory(hdfs_exec_logdir)
return_file = hdfs_exec_logdir + '/.hparams.json'
hdfs.dump(dumps(params), return_file)
return hdfs_exec_logdir, hdfs_experiment_dir
def delete(self, path, recursive=False):
return hopshdfs.delete(path, recursive=recursive)
def export(model_path, model_name, model_version=None, overwrite=False, metrics=None, description=None, synchronous=True, synchronous_timeout=120):
"""
Copies a trained model to the Models directory in the project and creates the directory structure of:
>>> Models
>>> |
>>> - model_name
>>> |
>>> - version_x
>>> |
>>> - version_y
For example if you run this:
>>> from hops import model
>>> model.export("iris_knn.pkl", "irisFlowerClassifier", metrics={'accuracy': accuracy})
It will copy the local model file "iris_knn.pkl" to /Projects/projectname/Models/irisFlowerClassifier/1/iris.knn.pkl
on HDFS, and overwrite in case there already exists a file with the same name in the directory.
If "model" is a directory on the local path exported by TensorFlow, and you run:
>>> model.export("/model", "mnist", metrics={'accuracy': accuracy, 'loss': loss})
It will copy the model directory contents to /Projects/projectname/Models/mnist/1/ , e.g the "model.pb" file and
the "variables" directory.
Args:
:model_path: path to the trained model (HDFS or local)
:model_name: name of the model
:model_version: version of the model
:overwrite: boolean flag whether to overwrite in case a model already exists in the exported directory
:metrics: dict of evaluation metrics to attach to model
:description: description about the model
:synchronous: whether to synchronously wait for the model to be indexed in the models rest endpoint
:synchronous_timeout: max timeout in seconds for waiting for the model to be indexed
Returns:
The path to where the model was exported
Raises:
:ValueError: if there was an error with th of the model due to invalid user input
:ModelNotFound: if the model was not found
"""
# Make sure model name is a string, users could supply numbers
model_name = str(model_name)
if not isinstance(model_path, string_types):
model_path = model_path.decode()
if not description:
description = 'A collection of models for ' + model_name
project_path = hdfs.project_path()
assert hdfs.exists(project_path + "Models"), "Your project is missing a dataset named Models, please create it."
if not hdfs.exists(model_path) and not os.path.exists(model_path):
raise ValueError("the provided model_path: {} , does not exist in HDFS or on the local filesystem".format(
model_path))
# make sure metrics are numbers
if metrics:
_validate_metadata(metrics)
model_dir_hdfs = project_path + constants.MODEL_SERVING.MODELS_DATASET + \
constants.DELIMITERS.SLASH_DELIMITER + model_name + constants.DELIMITERS.SLASH_DELIMITER
if not hdfs.exists(model_dir_hdfs):
hdfs.mkdir(model_dir_hdfs)
hdfs.chmod(model_dir_hdfs, "ug+rwx")
# User did not specify model_version, pick the current highest version + 1, set to 1 if no model exists
version_list = []
if not model_version and hdfs.exists(model_dir_hdfs):
model_version_directories = hdfs.ls(model_dir_hdfs)
for version_dir in model_version_directories:
try:
if hdfs.isdir(version_dir):
version_list.append(int(version_dir[len(model_dir_hdfs):]))
except:
pass
if len(version_list) > 0:
model_version = max(version_list) + 1
if not model_version:
model_version = 1
# Path to directory in HDFS to put the model files
model_version_dir_hdfs = model_dir_hdfs + str(model_version)
# If version directory already exists and we are not overwriting it then fail
if not overwrite and hdfs.exists(model_version_dir_hdfs):
raise ValueError("Could not create model directory: {}, the path already exists, "
"set flag overwrite=True "
"to remove the version directory and create the correct directory structure".format(model_version_dir_hdfs))
# Overwrite version directory by deleting all content (this is needed for Provenance to register Model as deleted)
if overwrite and hdfs.exists(model_version_dir_hdfs):
hdfs.delete(model_version_dir_hdfs, recursive=True)
hdfs.mkdir(model_version_dir_hdfs)
# At this point we can create the version directory if it does not exists
if not hdfs.exists(model_version_dir_hdfs):
hdfs.mkdir(model_version_dir_hdfs)
# Export the model files
if os.path.exists(model_path):
export_dir=_export_local_model(model_path, model_version_dir_hdfs, overwrite)
else:
export_dir=_export_hdfs_model(model_path, model_version_dir_hdfs, overwrite)
print("Exported model " + model_name + " as version " + str(model_version) + " successfully.")
jobName=None
if constants.ENV_VARIABLES.JOB_NAME_ENV_VAR in os.environ:
jobName = os.environ[constants.ENV_VARIABLES.JOB_NAME_ENV_VAR]
kernelId=None
if constants.ENV_VARIABLES.KERNEL_ID_ENV_VAR in os.environ:
kernelId = os.environ[constants.ENV_VARIABLES.KERNEL_ID_ENV_VAR]
# Attach modelName_modelVersion to experiment directory
model_summary = {'name': model_name, 'version': model_version, 'metrics': metrics,
'experimentId': None, 'description': description, 'jobName': jobName, 'kernelId': kernelId}
if 'ML_ID' in os.environ:
# Attach link from experiment to model
experiment_utils._attach_model_link_xattr(os.environ['ML_ID'], model_name + '_' + str(model_version))
# Attach model metadata to models version folder
model_summary['experimentId'] = os.environ['ML_ID']
experiment_utils._attach_model_xattr(model_name + "_" + str(model_version), experiment_utils.dumps(model_summary))
else:
experiment_utils._attach_model_xattr(model_name + "_" + str(model_version), experiment_utils.dumps(model_summary))
# Model metadata is attached asynchronously by Epipe, therefore this necessary to ensure following steps in a pipeline will not fail
if synchronous:
start_time = time.time()
sleep_seconds = 5
for i in range(int(synchronous_timeout/sleep_seconds)):
try:
time.sleep(sleep_seconds)
print("Polling " + model_name + " version " + str(model_version) + " for model availability.")
resp = get_model(model_name, model_version)
if resp.ok:
print("Model now available.")
return
print(model_name + " not ready yet, retrying in " + str(sleep_seconds) + " seconds.")
except ModelNotFound:
pass
print("Model not available during polling, set a higher value for synchronous_timeout to wait longer.")
return export_dir
def export(model_path, model_name, model_version=1, overwrite=False):
"""
Copies a trained model to the Models directory in the project and creates the directory structure of:
>>> Models
>>> |
>>> - model_name
>>> |
>>> - version_x
>>> |
>>> - version_y
For example if you run this:
>>> serving.export("iris_knn.pkl", "irisFlowerClassifier", 1, overwrite=True)
it will copy the local model file "iris_knn.pkl" to /Projects/projectname/Models/irisFlowerClassifier/1/iris.knn.pkl
on HDFS, and overwrite in case there already exists a file with the same name in the directory.
If you run:
>>> serving.export("Resources/iris_knn.pkl", "irisFlowerClassifier", 1, overwrite=True)
it will first check if the path Resources/iris_knn.pkl exists on your local filesystem in the current working
directory. If the path was not found, it will check in your project's HDFS directory and if it finds the model there
it will copy it to /Projects/projectname/Models/irisFlowerClassifier/1/iris.knn.pkl
If "model" is a directory on the local path exported by tensorflow, and you run:
:
>>> serving.export("/model/", "mnist", 1, overwrite=True)
It will copy the model directory contents to /Projects/projectname/Models/mnist/1/ , e.g the "model.pb" file and
the "variables" directory.
Args:
:model_path: path to the trained model (HDFS or local)
:model_name: name of the model/serving
:model_version: version of the model/serving
:overwrite: boolean flag whether to overwrite in case a serving already exists in the exported directory
Returns:
The path to where the model was exported
Raises:
:ValueError: if there was an error with the exportation of the model due to invalid user input
"""
if not hdfs.exists(model_path) and not os.path.exists(model_path):
raise ValueError("the provided model_path: {} , does not exist in HDFS or on the local filesystem".format(
model_path))
# Create directory in HDFS to put the model files
project_path = hdfs.project_path()
model_dir_hdfs = project_path + constants.MODEL_SERVING.MODELS_DATASET + \
constants.DELIMITERS.SLASH_DELIMITER + str(model_name) + \
constants.DELIMITERS.SLASH_DELIMITER + str(model_version) + \
constants.DELIMITERS.SLASH_DELIMITER
if not hdfs.exists(model_dir_hdfs):
hdfs.mkdir(model_dir_hdfs)
if (not overwrite) and hdfs.exists(model_dir_hdfs) and hdfs.isfile(model_dir_hdfs):
raise ValueError("Could not create model directory: {}, the path already exists and is a file, "
"set flag overwrite=True "
"to remove the file and create the correct directory structure".format(model_dir_hdfs))
if overwrite and hdfs.exists(model_dir_hdfs) and hdfs.isfile(model_dir_hdfs):
hdfs.delete(model_dir_hdfs)
hdfs.mkdir(model_dir_hdfs)
# Export the model files
if os.path.exists(model_path):
return _export_local_model(model_path, model_dir_hdfs, overwrite)
else:
return _export_hdfs_model(model_path, model_dir_hdfs, overwrite)