def get_model(model_id):
try:
model = model_business.get_by_model_id(ObjectId(model_id))
model = json_utility.convert_to_json(model.to_mongo())
except Exception as e:
return jsonify({'response': '%s: %s' % (str(Exception), e.args)}), 400
return jsonify({'response': model})
def _update_model():
GNN = {
"name": "General Neural Network",
"description": "keras_seq from keras",
"target_py_code": "server3/lib/models/keras_seq.py",
"entry_function": "keras_seq",
"to_code_function": "keras_seq_to_str",
"category": 0,
"model_type": 1,
"steps": models.KERAS_SEQ_STEPS,
"parameter_spec": models.KERAS_SEQ_SPEC,
"input": {
"type": "ndarray",
"n": None
}
}
MLP = {
"name": "Multilayer Perceptron",
"description": "Multilayer Perceptron (MLP) for multi-class softmax classification",
"target_py_code": "server3/lib/models/mlp.py",
"entry_function": "mlp",
"to_code_function": "mlp_to_str",
"category": 0,
"model_type": 1,
"steps": models.MLP_STEPS,
"parameter_spec": models.MLP,
"input": {
"type": "ndarray",
"n": None
}
}
LinearRegressor = {
"name": "Linear Regressor",
"description": "Custom linear regression model",
"target_py_code": "server3/lib/models/linear_regressor.py",
"entry_function": "linear_regressor_model_fn",
"to_code_function": "custom_model_to_str",
"category": 1,
"model_type": 0,
"steps": models.LinearRegressorSteps,
"parameter_spec": models.LinearRegressor,
"input": {
"type": "DataFrame"
}
}
LinearClassifier = {
"name": "Linear Classifier",
"description": "Custom linear classifier model",
"target_py_code": "server3/lib/models/linear_classifier.py",
"entry_function": "linear_classifier_model_fn",
"to_code_function": "custom_model_to_str",
"category": 1,
"model_type": 1,
"steps": models.LinearClassifierSteps,
"parameter_spec": models.LinearClassifier,
"input": {
"type": "DataFrame"
}
}
SVM = {
"name": "Support Vector Machine",
"description": "custom sdca model",
"target_py_code": "server3/lib/models/svm",
"entry_function": "svm_model_fn",
"to_code_function": "custom_model_to_str",
"category": 1,
"model_type": 1,
"steps": models.SVMSteps,
"parameter_spec": models.SVM,
"input": {
"type": "DataFrame"
}
}
RandomForest = {
"name": "Random Forest",
"description": "custom Random Forest model",
"target_py_code": "server3/lib/models/randomforest.py",
"entry_function": "random_forest_model_fn",
"to_code_function": "custom_model_to_str",
"category": 1,
"model_type": 1,
"steps": models.RandomForestSteps,
"parameter_spec": models.RandomForest,
"input": {
"type": "DataFrame"
}
}
GMM = {
"name": "Gaussian Mixture Models Cluster",
"description": "custom Gaussian Mixture Models Cluster",
"target_py_code": "server3/lib/models/gmm_cluster.py",
"entry_function": "gmm_cluster_model_fn",
"to_code_function": "custom_model_to_str",
"category": 2,
"model_type": 2,
"steps": models.GMMSteps,
"parameter_spec": models.GMMCluster,
"input": {
"type": "DataFrame"
}
}
Kmeans = {
"name": "Kmeans Clustering",
"description": "custom kmean model",
"target_py_code": "server3/lib/models/kmean",
"entry_function": "kmeans_cluster_model_fn",
"to_code_function": "custom_model_to_str",
"category": 2,
"model_type": 2,
"steps": models.KmeansSteps,
"parameter_spec": models.KmeansCluster,
"input": {
"type": "DataFrame"
}
}
HyperLinearRegressor = {
"name": "Hyper Linear Regressor",
"description": "Linear Regressor with hyper parameters tuning",
"target_py_code": "server3/lib/models/linear_regressor_hyper.py",
"entry_function": "linear_regression_hyper_model_fn",
"to_code_function": "custom_model_to_str",
"category": 7,
"model_type": 0,
"steps": models.LinearRegressorHyperSteps,
"parameter_spec": models.LinearRegressor,
"input": {
"type": "DataFrame"
}
}
user = UserBusiness.get_by_user_ID('system')
MODEL_DICT = [
{
"_id": ObjectId("598293510c11f34ca9a2b486"),
"is_private": False,
"user_ID": user.user_ID,
"obj": GNN
},
{
"_id": ObjectId("597ae03e0c11f32f859a0f8c"),
"is_private": False,
"user_ID": user.user_ID,
"obj": LinearRegressor
},
{
"_id": ObjectId("5980378d0c11f318f61ce18a"),
"is_private": False,
"user_ID": user.user_ID,
"obj": LinearClassifier
},
{
"_id": ObjectId("596f5c8bd123ab59405c6e11"),
"is_private": False,
"user_ID": user.user_ID,
"obj": MLP
},
{
"_id": ObjectId("59687821d123abcfbfe8cab9"),
"is_private": False,
"user_ID": user.user_ID,
"obj": SVM
},
{
"_id": ObjectId("5983e7e339b6bd0c28bdbb29"),
"is_private": False,
"user_ID": user.user_ID,
"obj": RandomForest
},
{
"_id": ObjectId("59faffbb0c11f30876f25f07"),
"is_private": False,
"user_ID": user.user_ID,
"obj": GMM
},
{
"_id": ObjectId("59687821d123abcfbfe8cabb"),
"is_private": False,
"user_ID": user.user_ID,
"obj": Kmeans
},
{
"_id": ObjectId("5a0408f0d845c03b92e7bb35"),
"is_private": False,
"user_ID": user.user_ID,
"obj": HyperLinearRegressor
},
]
for model in MODEL_DICT:
try:
model_obj = model_business.get_by_model_id(model['_id'])
except DoesNotExist:
# create model
model.pop('_id')
obj = {
**model.pop('obj'),
**model
}
print('Added:', add_model_with_ownership(**obj))
except KeyError:
obj = {
**model.pop('obj'),
**model
}
print('Added:', add_model_with_ownership(**obj))
else:
if model_obj:
# update model
model_id = model.get('_id')
print('Updated:', model_business.update_by_id(model_id,
**model.get(
'obj')).name)
def run_model(conf, project_id, data_source_id, model_id, job_id, **kwargs):
"""
run model by model_id and the parameter config
:param conf:
:param project_id:
:param data_source_id:
:param model_id:
:param job_id:
:param kwargs:
:return:
"""
model = model_business.get_by_model_id(model_id)
project = project_business.get_by_id(project_id)
ownership = ownership_business.get_ownership_by_owned_item(project,
'project')
result_dir = os.path.join(user_directory, ownership.user.user_ID,
project.name, job_id)
# import model function
if model['category'] == ModelType['neural_network']:
# keras nn
f = getattr(models, model.entry_function)
input_dict = manage_nn_input(conf, data_source_id, **kwargs)
return job_service.run_code(conf, project_id, data_source_id,
model, f, job_id, input_dict,
result_dir=result_dir)
elif model['category'] == ModelType['unstructured']:
# input from folder
f = getattr(models, model.entry_function)
input_dict = model_input_manager_unstructured(conf, data_source_id,
**kwargs)
return job_service.run_code(conf, project_id, None,
model, f, job_id, input_dict,
file_id=data_source_id,
result_dir=result_dir)
elif model['category'] == ModelType['hyperopt']:
f = getattr(models, model.entry_function)
fit = conf.get('fit', None)
data_fields = fit.get('data_fields', [[], []])
input_dict = model_input_manager_custom_supervised(data_fields,
data_source_id,
model.name,
**kwargs)
return job_service.run_code(conf, project_id, data_source_id,
model, f, job_id, input_dict,
result_dir=result_dir)
else:
# custom models
f = models.custom_model
model_fn = getattr(models, model.entry_function)
fit = conf.get('fit', None)
if model['category'] == ModelType['custom_supervised']:
data_fields = fit.get('data_fields', [[], []])
input_dict = model_input_manager_custom_supervised(data_fields,
data_source_id,
model.name,
**kwargs)
return job_service.run_code(conf, project_id, data_source_id,
model, f, job_id, model_fn, input_dict,
result_dir=result_dir)
if model['category'] == ModelType['unsupervised']:
x_cols = fit.get('data_fields', [])
input_dict = model_input_manager_unsupervised(x_cols,
data_source_id,
model.name,
**kwargs)
return job_service.run_code(conf, project_id, data_source_id,
model, f, job_id, model_fn, input_dict,
result_dir=result_dir)
if model['category'] == ModelType['hyperopt']:
data_fields = fit.get('data_fields', [[], []])
input_dict = model_input_manager_custom_supervised(data_fields,
data_source_id,
model.name,
**kwargs)
return job_service.run_code(conf, project_id, data_source_id,
model, f, job_id, model_fn, input_dict,
result_dir=result_dir)
def model_to_code(conf, project_id, data_source_id, model_id, job_obj,
**kwargs):
"""
run model by model_id and the parameter config
:param conf:
:param project_id:
:param data_source_id:
:param model_id:
:param kwargs:
:return:
"""
file_id = kwargs.get('file_id')
staging_data_set_obj = None
if data_source_id:
staging_data_set_obj = \
staging_data_set_business.get_by_id(data_source_id)
project_obj = project_business.get_by_id(project_id)
file_dict = {'file': ObjectId(file_id)} if file_id else {}
model_obj = model_business.get_by_model_id(model_id)
run_args = {
"conf": conf,
"project_id": project_id,
"data_source_id": data_source_id,
"model_id": model_id,
"kwargs": kwargs
}
# # create model job
# job_obj = job_business.add_model_job(model_obj, staging_data_set_obj,
# project_obj, params=conf,
# run_args=run_args,
# **file_dict)
job_obj = job_business.update_job_by_id(job_obj.id, model=model_obj,
staging_data_set=staging_data_set_obj,
project=project_obj, params=conf,
run_args=run_args, status=100)
job_id = str(job_obj.id)
# model_obj = model_business.get_by_model_id(model_id)
f = getattr(models, model_obj.to_code_function)
if model_obj['category'] == 0:
# keras nn
head_str = manage_supervised_input_to_str(conf, data_source_id,
**kwargs)
return job_service.run_code(conf, project_id, data_source_id,
model_obj, f, job_id, head_str)
elif model_obj['category'] == ModelType['unstructured']:
# input from folder
head_str = manage_unstructured_to_str(conf, data_source_id,
**kwargs)
return job_service.run_code(conf, project_id, None,
model_obj, f, job_id, head_str,
file_id=data_source_id)
elif model_obj['category'] == ModelType['advanced']:
# no input
return job_service.run_code(conf, project_id, None,
model_obj, f, job_id, '',
file_id=None)
else:
# custom models
head_str = ''
head_str += 'import logging\n'
head_str += 'import numpy as np\n'
head_str += 'import pandas as pd\n'
head_str += 'import tensorflow as tf\n'
head_str += 'from tensorflow.python.framework import constant_op\n'
head_str += 'from tensorflow.python.framework import dtypes\n'
head_str += 'from tensorflow.contrib.learn.python.learn import metric_spec\n'
head_str += 'from server3.lib import models\n'
head_str += 'from server3.lib.models.modified_tf_file.monitors import ValidationMonitor\n'
head_str += 'from server3.business import staging_data_set_business\n'
head_str += 'from server3.business import staging_data_business\n'
head_str += 'from server3.service import staging_data_service\n'
head_str += "from server3.service import job_service\n"
head_str += 'from server3.service.model_service import ' \
'split_categorical_and_continuous\n'
head_str += 'from server3.service.custom_log_handler ' \
'import MetricsHandler\n'
head_str += 'model_fn = models.%s\n' % model_obj.entry_function
head_str += "data_source_id = '%s'\n" % data_source_id
head_str += "model_name = '%s'\n" % model_obj.name
head_str += "kwargs = %s\n" % kwargs
fit = conf.get('fit', None)
if model_obj['category'] == 1:
data_fields = fit.get('data_fields', [[], []])
head_str += 'data_fields = %s\n' % data_fields
head_str += inspect.getsource(
model_input_manager_custom_supervised)
head_str += "input_dict = model_input_manager_custom_supervised(" \
"data_fields, data_source_id, model_name, **kwargs)\n"
elif model_obj['category'] == 2:
x_cols = fit.get('data_fields', [])
head_str += "x_cols = %s\n" % x_cols
head_str += inspect.getsource(model_input_manager_unsupervised)
head_str += "input_dict = model_input_manager_unsupervised(x_cols, " \
"data_source_id, model_name)\n"
return job_service.run_code(conf, project_id, data_source_id,
model_obj, f, job_id, head_str)
def kube_run_model(conf, project_id, data_source_id, model_id, job_obj,
**kwargs):
# file_id = kwargs.get('file_id')
staging_data_set_obj = None
if data_source_id:
staging_data_set_obj = \
staging_data_set_business.get_by_id(data_source_id)
project_obj = project_business.get_by_id(project_id)
# file_dict = {'file': ObjectId(file_id)} if file_id else {}
model_obj = model_business.get_by_model_id(model_id)
run_args = {
"conf": conf,
"project_id": project_id,
"data_source_id": data_source_id,
"model_id": model_id,
"kwargs": kwargs
}
job_obj = job_business.update_job_by_id(job_obj.id, model=model_obj,
staging_data_set=staging_data_set_obj,
project=project_obj, params=conf,
run_args=run_args, status=100)
job_id = str(job_obj.id)
print(job_id)
return run_model(conf, project_id, data_source_id, model_id, job_id,
**kwargs)
cwd = os.getcwd()
job_name = job_id + '-training-job'
client = kube_service.client
try:
# TODO need to terminate running pod
kube_service.delete_job(job_name)
while True:
kube_service.get_job(job_name)
time.sleep(1)
except client.rest.ApiException:
print('job not exists or deleted, ok to create')
kube_json = {
"apiVersion": "batch/v1",
"kind": "Job",
"metadata": {
"name": job_name
},
"spec": {
"template": {
"metadata": {
"labels": {
"app": job_id
}
},
"spec": {
"containers": [
{
"name": job_id,
"image": "10.52.14.192/gzyw/model_app_pre",
"imagePullPolicy": "IfNotPresent",
"securityContext": {
"privileged": True,
},
"stdin": True,
"command": ["/usr/local/bin/python"],
"args": [
"run_model.py",
"--job_id", job_id
],
"volumeMounts": [
{
"mountPath": "/pyserver/user_directory",
"name": "nfsvol"
},
]
}
],
"restartPolicy": "Never",
# "activeDeadlineSeconds": 1,
"volumes": [
{
"name": "nfsvol",
"persistentVolumeClaim": {
"claimName": "nfs-pvc"
}
},
]
},
},
}
}
# file_utils.write_to_filepath(json.dumps(kube_json), './model_app.json')
# return
api = kube_service.job_api
resp = api.create_namespaced_job(body=kube_json, namespace=NAMESPACE)
print("Job created. status='%s'" % str(resp.status))
return {'job_id': job_id}