def first_save_to_db(user_ID, name, description, input_info, output_info,
examples, version,
deploy_name, server,
input_type, model_base_path, job,
job_id, model_name, related_fields,
related_tasks, tags, is_private, data_fields,
input_data_demo_string,
service_name,projectId,
**optional):
"""
:param user_ID:
:param name:
:param description:
:param input_info:
:param output_info:
:param examples:
:param version:
:param deploy_name:
:param server:
:param input_type:
:param model_base_path:
:param job:
:param job_id:
:param is_private:
:param service_name:
:param model_name:
:param optional:
:return:
"""
user = user_business.get_by_user_ID(user_ID)
served_model = served_model_business.add(name, description, input_info,
output_info,
examples, version, deploy_name,
server, input_type,
model_base_path, job,
service_name, model_name,
related_fields,
related_tasks,
tags, is_private, data_fields,
input_data_demo_string,
create_time=datetime.utcnow(),
user=user, user_ID=user_ID,
projectId=projectId,
**optional)
ownership_business.add(user, is_private, served_model=served_model)
job_business.update_job_by_id(job_id, served_model=served_model)
return served_model
def update_job(job_id):
data = request.get_json()
new_job = job_business.update_job_by_id(job_id, **data)
new_job = json_utility.convert_to_json(new_job.to_mongo())
return jsonify({"response": new_job}), 200
def model_to_code(conf, project_id, data_source_id, model_id, job_obj,
**kwargs):
"""
run model by model_id and the parameter config
:param conf:
:param project_id:
:param data_source_id:
:param model_id:
:param kwargs:
:return:
"""
file_id = kwargs.get('file_id')
staging_data_set_obj = None
if data_source_id:
staging_data_set_obj = \
staging_data_set_business.get_by_id(data_source_id)
project_obj = project_business.get_by_id(project_id)
file_dict = {'file': ObjectId(file_id)} if file_id else {}
model_obj = model_business.get_by_model_id(model_id)
run_args = {
"conf": conf,
"project_id": project_id,
"data_source_id": data_source_id,
"model_id": model_id,
"kwargs": kwargs
}
# # create model job
# job_obj = job_business.add_model_job(model_obj, staging_data_set_obj,
# project_obj, params=conf,
# run_args=run_args,
# **file_dict)
job_obj = job_business.update_job_by_id(job_obj.id, model=model_obj,
staging_data_set=staging_data_set_obj,
project=project_obj, params=conf,
run_args=run_args, status=100)
job_id = str(job_obj.id)
# model_obj = model_business.get_by_model_id(model_id)
f = getattr(models, model_obj.to_code_function)
if model_obj['category'] == 0:
# keras nn
head_str = manage_supervised_input_to_str(conf, data_source_id,
**kwargs)
return job_service.run_code(conf, project_id, data_source_id,
model_obj, f, job_id, head_str)
elif model_obj['category'] == ModelType['unstructured']:
# input from folder
head_str = manage_unstructured_to_str(conf, data_source_id,
**kwargs)
return job_service.run_code(conf, project_id, None,
model_obj, f, job_id, head_str,
file_id=data_source_id)
elif model_obj['category'] == ModelType['advanced']:
# no input
return job_service.run_code(conf, project_id, None,
model_obj, f, job_id, '',
file_id=None)
else:
# custom models
head_str = ''
head_str += 'import logging\n'
head_str += 'import numpy as np\n'
head_str += 'import pandas as pd\n'
head_str += 'import tensorflow as tf\n'
head_str += 'from tensorflow.python.framework import constant_op\n'
head_str += 'from tensorflow.python.framework import dtypes\n'
head_str += 'from tensorflow.contrib.learn.python.learn import metric_spec\n'
head_str += 'from server3.lib import models\n'
head_str += 'from server3.lib.models.modified_tf_file.monitors import ValidationMonitor\n'
head_str += 'from server3.business import staging_data_set_business\n'
head_str += 'from server3.business import staging_data_business\n'
head_str += 'from server3.service import staging_data_service\n'
head_str += "from server3.service import job_service\n"
head_str += 'from server3.service.model_service import ' \
'split_categorical_and_continuous\n'
head_str += 'from server3.service.custom_log_handler ' \
'import MetricsHandler\n'
head_str += 'model_fn = models.%s\n' % model_obj.entry_function
head_str += "data_source_id = '%s'\n" % data_source_id
head_str += "model_name = '%s'\n" % model_obj.name
head_str += "kwargs = %s\n" % kwargs
fit = conf.get('fit', None)
if model_obj['category'] == 1:
data_fields = fit.get('data_fields', [[], []])
head_str += 'data_fields = %s\n' % data_fields
head_str += inspect.getsource(
model_input_manager_custom_supervised)
head_str += "input_dict = model_input_manager_custom_supervised(" \
"data_fields, data_source_id, model_name, **kwargs)\n"
elif model_obj['category'] == 2:
x_cols = fit.get('data_fields', [])
head_str += "x_cols = %s\n" % x_cols
head_str += inspect.getsource(model_input_manager_unsupervised)
head_str += "input_dict = model_input_manager_unsupervised(x_cols, " \
"data_source_id, model_name)\n"
return job_service.run_code(conf, project_id, data_source_id,
model_obj, f, job_id, head_str)
def update_job_status_fn(job, **result):
if job is None:
raise ValueError('no job id passed')
job_business.update_job_by_id(job['id'], **result)
def kube_run_model(conf, project_id, data_source_id, model_id, job_obj,
**kwargs):
# file_id = kwargs.get('file_id')
staging_data_set_obj = None
if data_source_id:
staging_data_set_obj = \
staging_data_set_business.get_by_id(data_source_id)
project_obj = project_business.get_by_id(project_id)
# file_dict = {'file': ObjectId(file_id)} if file_id else {}
model_obj = model_business.get_by_model_id(model_id)
run_args = {
"conf": conf,
"project_id": project_id,
"data_source_id": data_source_id,
"model_id": model_id,
"kwargs": kwargs
}
job_obj = job_business.update_job_by_id(job_obj.id, model=model_obj,
staging_data_set=staging_data_set_obj,
project=project_obj, params=conf,
run_args=run_args, status=100)
job_id = str(job_obj.id)
print(job_id)
return run_model(conf, project_id, data_source_id, model_id, job_id,
**kwargs)
cwd = os.getcwd()
job_name = job_id + '-training-job'
client = kube_service.client
try:
# TODO need to terminate running pod
kube_service.delete_job(job_name)
while True:
kube_service.get_job(job_name)
time.sleep(1)
except client.rest.ApiException:
print('job not exists or deleted, ok to create')
kube_json = {
"apiVersion": "batch/v1",
"kind": "Job",
"metadata": {
"name": job_name
},
"spec": {
"template": {
"metadata": {
"labels": {
"app": job_id
}
},
"spec": {
"containers": [
{
"name": job_id,
"image": "10.52.14.192/gzyw/model_app_pre",
"imagePullPolicy": "IfNotPresent",
"securityContext": {
"privileged": True,
},
"stdin": True,
"command": ["/usr/local/bin/python"],
"args": [
"run_model.py",
"--job_id", job_id
],
"volumeMounts": [
{
"mountPath": "/pyserver/user_directory",
"name": "nfsvol"
},
]
}
],
"restartPolicy": "Never",
# "activeDeadlineSeconds": 1,
"volumes": [
{
"name": "nfsvol",
"persistentVolumeClaim": {
"claimName": "nfs-pvc"
}
},
]
},
},
}
}
# file_utils.write_to_filepath(json.dumps(kube_json), './model_app.json')
# return
api = kube_service.job_api
resp = api.create_namespaced_job(body=kube_json, namespace=NAMESPACE)
print("Job created. status='%s'" % str(resp.status))
return {'job_id': job_id}