def _export_hdfs_model(hdfs_model_path, model_dir_hdfs, overwrite):
"""
Exports a hdfs directory of model files to Hopsworks "Models" dataset
Args:
:hdfs_model_path: the path to the model files in hdfs
:model_dir_hdfs: path to the directory in HDFS to put the model files
:overwrite: boolean flag whether to overwrite in case a model already exists in the exported directory
Returns:
the path to the exported model files in HDFS
"""
if hdfs.isdir(hdfs_model_path):
for file_source_path in hdfs.ls(hdfs_model_path):
model_name = file_source_path
if constants.DELIMITERS.SLASH_DELIMITER in file_source_path:
last_index = model_name.rfind(constants.DELIMITERS.SLASH_DELIMITER)
model_name = model_name[last_index + 1:]
dest_path = model_dir_hdfs + constants.DELIMITERS.SLASH_DELIMITER + model_name
hdfs.cp(file_source_path, dest_path, overwrite=overwrite)
elif hdfs.isfile(hdfs_model_path):
model_name = hdfs_model_path
if constants.DELIMITERS.SLASH_DELIMITER in hdfs_model_path:
last_index = model_name.rfind(constants.DELIMITERS.SLASH_DELIMITER)
model_name = model_name[last_index + 1:]
dest_path = model_dir_hdfs + constants.DELIMITERS.SLASH_DELIMITER + model_name
hdfs.cp(hdfs_model_path, dest_path, overwrite=overwrite)
return model_dir_hdfs
def export(model_path, model_name, model_version=1, overwrite=False):
"""
Copies a trained model to the Models directory in the project and creates the directory structure of:
>>> Models
>>> |
>>> - model_name
>>> |
>>> - version_x
>>> |
>>> - version_y
For example if you run this:
>>> serving.export("iris_knn.pkl", "irisFlowerClassifier", 1, overwrite=True)
it will copy the local model file "iris_knn.pkl" to /Projects/projectname/Models/irisFlowerClassifier/1/iris.knn.pkl
on HDFS, and overwrite in case there already exists a file with the same name in the directory.
If you run:
>>> serving.export("Resources/iris_knn.pkl", "irisFlowerClassifier", 1, overwrite=True)
it will first check if the path Resources/iris_knn.pkl exists on your local filesystem in the current working
directory. If the path was not found, it will check in your project's HDFS directory and if it finds the model there
it will copy it to /Projects/projectname/Models/irisFlowerClassifier/1/iris.knn.pkl
If "model" is a directory on the local path exported by tensorflow, and you run:
:
>>> serving.export("/model/", "mnist", 1, overwrite=True)
It will copy the model directory contents to /Projects/projectname/Models/mnist/1/ , e.g the "model.pb" file and
the "variables" directory.
Args:
:model_path: path to the trained model (HDFS or local)
:model_name: name of the model/serving
:model_version: version of the model/serving
:overwrite: boolean flag whether to overwrite in case a serving already exists in the exported directory
Returns:
The path to where the model was exported
Raises:
:ValueError: if there was an error with the exportation of the model due to invalid user input
"""
if not hdfs.exists(model_path) and not os.path.exists(model_path):
raise ValueError("the provided model_path: {} , does not exist in HDFS or on the local filesystem".format(
model_path))
# Create directory in HDFS to put the model files
project_path = hdfs.project_path()
model_dir_hdfs = project_path + constants.MODEL_SERVING.MODELS_DATASET + \
constants.DELIMITERS.SLASH_DELIMITER + str(model_name) + \
constants.DELIMITERS.SLASH_DELIMITER + str(model_version) + \
constants.DELIMITERS.SLASH_DELIMITER
if not hdfs.exists(model_dir_hdfs):
hdfs.mkdir(model_dir_hdfs)
if (not overwrite) and hdfs.exists(model_dir_hdfs) and hdfs.isfile(model_dir_hdfs):
raise ValueError("Could not create model directory: {}, the path already exists and is a file, "
"set flag overwrite=True "
"to remove the file and create the correct directory structure".format(model_dir_hdfs))
if overwrite and hdfs.exists(model_dir_hdfs) and hdfs.isfile(model_dir_hdfs):
hdfs.delete(model_dir_hdfs)
hdfs.mkdir(model_dir_hdfs)
# Export the model files
if os.path.exists(model_path):
return _export_local_model(model_path, model_dir_hdfs, overwrite)
else:
return _export_hdfs_model(model_path, model_dir_hdfs, overwrite)