def _export_hdfs_model(hdfs_model_path, model_dir_hdfs, overwrite):
"""
Exports a hdfs directory of model files to Hopsworks "Models" dataset
Args:
:hdfs_model_path: the path to the model files in hdfs
:model_dir_hdfs: path to the directory in HDFS to put the model files
:overwrite: boolean flag whether to overwrite in case a model already exists in the exported directory
Returns:
the path to the exported model files in HDFS
"""
if hdfs.isdir(hdfs_model_path):
for file_source_path in hdfs.ls(hdfs_model_path):
model_name = file_source_path
if constants.DELIMITERS.SLASH_DELIMITER in file_source_path:
last_index = model_name.rfind(constants.DELIMITERS.SLASH_DELIMITER)
model_name = model_name[last_index + 1:]
dest_path = model_dir_hdfs + constants.DELIMITERS.SLASH_DELIMITER + model_name
hdfs.cp(file_source_path, dest_path, overwrite=overwrite)
elif hdfs.isfile(hdfs_model_path):
model_name = hdfs_model_path
if constants.DELIMITERS.SLASH_DELIMITER in hdfs_model_path:
last_index = model_name.rfind(constants.DELIMITERS.SLASH_DELIMITER)
model_name = model_name[last_index + 1:]
dest_path = model_dir_hdfs + constants.DELIMITERS.SLASH_DELIMITER + model_name
hdfs.cp(hdfs_model_path, dest_path, overwrite=overwrite)
return model_dir_hdfs
def _clean_dir(clean_dir, keep=[]):
"""Deletes all files in a directory but keeps a few.
"""
if not hopshdfs.isdir(clean_dir):
raise ValueError(
"{} is not a directory. Use `hops.hdfs.delete()` to delete single "
"files.".format(clean_dir))
for path in hopshdfs.ls(clean_dir):
if path not in keep:
hopshdfs.delete(path, recursive=True)
def _build_summary_json(logdir):
"""Builds the summary json to be read by the experiments service.
"""
combinations = []
for trial in hopshdfs.ls(logdir):
if hopshdfs.isdir(trial):
return_file = trial + "/.outputs.json"
hparams_file = trial + "/.hparams.json"
if hopshdfs.exists(return_file) and hopshdfs.exists(hparams_file):
metric_arr = experiment_utils._convert_return_file_to_arr(
return_file)
hparams_dict = _load_hparams(hparams_file)
combinations.append({
"parameters": hparams_dict,
"outputs": metric_arr
})
return json.dumps({"combinations": combinations},
default=json_default_numpy)
def isdir(self, dir_path, project=None):
return hopshdfs.isdir(dir_path, project=project)
def export(model_path, model_name, model_version=None, overwrite=False, metrics=None, description=None, synchronous=True, synchronous_timeout=120):
"""
Copies a trained model to the Models directory in the project and creates the directory structure of:
>>> Models
>>> |
>>> - model_name
>>> |
>>> - version_x
>>> |
>>> - version_y
For example if you run this:
>>> from hops import model
>>> model.export("iris_knn.pkl", "irisFlowerClassifier", metrics={'accuracy': accuracy})
It will copy the local model file "iris_knn.pkl" to /Projects/projectname/Models/irisFlowerClassifier/1/iris.knn.pkl
on HDFS, and overwrite in case there already exists a file with the same name in the directory.
If "model" is a directory on the local path exported by TensorFlow, and you run:
>>> model.export("/model", "mnist", metrics={'accuracy': accuracy, 'loss': loss})
It will copy the model directory contents to /Projects/projectname/Models/mnist/1/ , e.g the "model.pb" file and
the "variables" directory.
Args:
:model_path: path to the trained model (HDFS or local)
:model_name: name of the model
:model_version: version of the model
:overwrite: boolean flag whether to overwrite in case a model already exists in the exported directory
:metrics: dict of evaluation metrics to attach to model
:description: description about the model
:synchronous: whether to synchronously wait for the model to be indexed in the models rest endpoint
:synchronous_timeout: max timeout in seconds for waiting for the model to be indexed
Returns:
The path to where the model was exported
Raises:
:ValueError: if there was an error with th of the model due to invalid user input
:ModelNotFound: if the model was not found
"""
# Make sure model name is a string, users could supply numbers
model_name = str(model_name)
if not isinstance(model_path, string_types):
model_path = model_path.decode()
if not description:
description = 'A collection of models for ' + model_name
project_path = hdfs.project_path()
assert hdfs.exists(project_path + "Models"), "Your project is missing a dataset named Models, please create it."
if not hdfs.exists(model_path) and not os.path.exists(model_path):
raise ValueError("the provided model_path: {} , does not exist in HDFS or on the local filesystem".format(
model_path))
# make sure metrics are numbers
if metrics:
_validate_metadata(metrics)
model_dir_hdfs = project_path + constants.MODEL_SERVING.MODELS_DATASET + \
constants.DELIMITERS.SLASH_DELIMITER + model_name + constants.DELIMITERS.SLASH_DELIMITER
if not hdfs.exists(model_dir_hdfs):
hdfs.mkdir(model_dir_hdfs)
hdfs.chmod(model_dir_hdfs, "ug+rwx")
# User did not specify model_version, pick the current highest version + 1, set to 1 if no model exists
version_list = []
if not model_version and hdfs.exists(model_dir_hdfs):
model_version_directories = hdfs.ls(model_dir_hdfs)
for version_dir in model_version_directories:
try:
if hdfs.isdir(version_dir):
version_list.append(int(version_dir[len(model_dir_hdfs):]))
except:
pass
if len(version_list) > 0:
model_version = max(version_list) + 1
if not model_version:
model_version = 1
# Path to directory in HDFS to put the model files
model_version_dir_hdfs = model_dir_hdfs + str(model_version)
# If version directory already exists and we are not overwriting it then fail
if not overwrite and hdfs.exists(model_version_dir_hdfs):
raise ValueError("Could not create model directory: {}, the path already exists, "
"set flag overwrite=True "
"to remove the version directory and create the correct directory structure".format(model_version_dir_hdfs))
# Overwrite version directory by deleting all content (this is needed for Provenance to register Model as deleted)
if overwrite and hdfs.exists(model_version_dir_hdfs):
hdfs.delete(model_version_dir_hdfs, recursive=True)
hdfs.mkdir(model_version_dir_hdfs)
# At this point we can create the version directory if it does not exists
if not hdfs.exists(model_version_dir_hdfs):
hdfs.mkdir(model_version_dir_hdfs)
# Export the model files
if os.path.exists(model_path):
export_dir=_export_local_model(model_path, model_version_dir_hdfs, overwrite)
else:
export_dir=_export_hdfs_model(model_path, model_version_dir_hdfs, overwrite)
print("Exported model " + model_name + " as version " + str(model_version) + " successfully.")
jobName=None
if constants.ENV_VARIABLES.JOB_NAME_ENV_VAR in os.environ:
jobName = os.environ[constants.ENV_VARIABLES.JOB_NAME_ENV_VAR]
kernelId=None
if constants.ENV_VARIABLES.KERNEL_ID_ENV_VAR in os.environ:
kernelId = os.environ[constants.ENV_VARIABLES.KERNEL_ID_ENV_VAR]
# Attach modelName_modelVersion to experiment directory
model_summary = {'name': model_name, 'version': model_version, 'metrics': metrics,
'experimentId': None, 'description': description, 'jobName': jobName, 'kernelId': kernelId}
if 'ML_ID' in os.environ:
# Attach link from experiment to model
experiment_utils._attach_model_link_xattr(os.environ['ML_ID'], model_name + '_' + str(model_version))
# Attach model metadata to models version folder
model_summary['experimentId'] = os.environ['ML_ID']
experiment_utils._attach_model_xattr(model_name + "_" + str(model_version), experiment_utils.dumps(model_summary))
else:
experiment_utils._attach_model_xattr(model_name + "_" + str(model_version), experiment_utils.dumps(model_summary))
# Model metadata is attached asynchronously by Epipe, therefore this necessary to ensure following steps in a pipeline will not fail
if synchronous:
start_time = time.time()
sleep_seconds = 5
for i in range(int(synchronous_timeout/sleep_seconds)):
try:
time.sleep(sleep_seconds)
print("Polling " + model_name + " version " + str(model_version) + " for model availability.")
resp = get_model(model_name, model_version)
if resp.ok:
print("Model now available.")
return
print(model_name + " not ready yet, retrying in " + str(sleep_seconds) + " seconds.")
except ModelNotFound:
pass
print("Model not available during polling, set a higher value for synchronous_timeout to wait longer.")
return export_dir