def main(unused_argv):
job_id = FLAGS.job_id
if job_id == "59ae047e0c11f35fafebc422":
raise ValueError('no job_id flag')
job = job_business.get_by_job_id(job_id)
# project id
project_id = job.project.id
project = project_business.get_by_id(project_id)
ow = ownership_business.get_ownership_by_owned_item(project, 'project')
# user ID
user_ID = ow.user.user_ID
args = job.run_args
try:
run_model(args['conf'], args['project_id'], args['data_source_id'],
args['model_id'], job_id, **args['kwargs'])
except Exception:
# if error send error, save error and raise error
exc_type, exc_value, exc_traceback = sys.exc_info()
message = {
'error': repr(traceback.format_exception(exc_type, exc_value,
exc_traceback)),
'type': 'model'
}
print(message)
emit_error(message, str(project_id), job_id=job_id, user_ID=user_ID)
save_job_status(job, error=message, status=300)
else:
message = {
'project_name': project.name,
'type': 'model',
'complete': True,
'content': 'Model job completed in project ' + project.name
}
emit_success(message, str(project_id), job_id=job_id, user_ID=user_ID)
def save_as_result(job_id, new_sds_name):
job_obj = job_business.get_by_job_id(job_id)
result = job_obj.result
toolkit = job_obj.toolkit
project_obj = job_obj.project
sds_id = staging_data_set_business.add(
name=new_sds_name,
description='des',
project=project_obj,
# job=job_obj
)
# 拿到原表
old_sds = StepBusiness.get_datasource(job_obj.steps)
table = staging_data_business.get_by_staging_data_set_id(old_sds)
table_dict = []
for i in range(len(table)):
row = table[i].to_mongo().to_dict()
row.pop("_id")
row.pop("staging_data_set")
table_dict.append(row)
# 复制原表
staging_data_business.add_many(staging_data_set=sds_id,
data_array=table_dict)
# 保存结果
save_result_sub(result, sds_id, toolkit)
def save_result(job_id):
job_obj = job_business.get_by_job_id(job_id)
result = job_obj.result
toolkit = job_obj.toolkit
sds_id = StepBusiness.get_datasource(job_obj.steps)
save_result_sub(result, sds_id, toolkit)
def run_job():
data = request.get_json()
job_id = data['section_id']
project_id = data["project_id"]
job_obj = job_business.get_by_job_id(job_id)
project = project_business.get_by_id(project_id)
ow = ownership_business.get_ownership_by_owned_item(project, 'project')
# user ID
user_ID = ow.user.user_ID
type = None
try:
if job_obj.toolkit:
type = 'toolkit'
complete = True
content = 'Toolkit job completed in project ' + project.name
result = job_service.run_toolkit_job(project_id=project_id,
job_obj=job_obj)
elif job_obj.model:
type = 'model'
complete = False
content = 'Model job successfully created in project ' + \
project.name
result = job_service.run_model_job(project_id=project_id,
job_obj=job_obj)
else:
return jsonify({"response":
'no model and toolkit in job object'}), 400
result = json_utility.convert_to_json(result)
except Exception as e:
# if error send error, save error and raise error
exc_type, exc_value, exc_traceback = sys.exc_info()
message = {
'error':
repr(traceback.format_exception(exc_type, exc_value,
exc_traceback)),
'type':
type
}
print(message)
emit_error(message, str(project_id), job_id=job_id, user_ID=user_ID)
save_job_status(job_obj, error=message, status=300)
raise e
# return jsonify({
# "response": {
# "result": message
# }}), 200
else:
message = {
'project_name': project.name,
'type': type,
'complete': complete,
'content': content
}
emit_success(message, str(project_id), job_id=job_id, user_ID=user_ID)
return jsonify({"response": {"result": result}}), 200
def wrapper(*args, **kw):
# create a job
# model_obj = model_business.get_by_model_id(model_id)
result_dir = kwargs.get('result_dir')
project_obj = project_business.get_by_id(project_id)
job_obj = job_business.get_by_job_id(job_id)
# update a project
project_business.insert_job_by_id(project_id, job_obj.id)
project_business.update_items_to_list_field(project_id,
related_tasks=TYPE.get(
model_obj.category,
[]))
# create result sds for model
sds_name = '%s_%s_result' % (model_obj['name'], job_obj['id'])
try:
sds = staging_data_set_business.get_by_job_id(job_obj.id)
except DoesNotExist:
print('free to create sds')
else:
staging_data_set_business.remove_by_id(sds.id)
finally:
result_sds_obj = staging_data_set_business.add(sds_name,
'des',
project_obj,
job=job_obj,
type='result')
# run
if result_dir:
# result_dir += str(job_obj['id']) + '/'
try:
os.makedirs(result_dir)
except FileExistsError:
print('dir exists, no need to create')
kw['result_dir'] = result_dir
# generate_job_py(func, *args, **kw, result_sds=result_sds_obj,
# project_id=project_id)
func_result = func(*args,
**kw,
result_sds=result_sds_obj,
project_id=project_id,
job_id=job_id)
# update a job
job_business.end_job(job_obj)
if isinstance(func_result, dict):
func_result['job_id'] = str(job_obj['id'])
return func_result
def to_code():
data = request.get_json()
job_id = data['section_id']
project_id = data["project_id"]
job_obj = job_business.get_by_job_id(job_id)
project = project_business.get_by_id(project_id)
ow = ownership_business.get_ownership_by_owned_item(project, 'project')
# user ID
user_ID = ow.user.user_ID
code = job_service.model_job_to_code(project_id=project_id,
job_obj=job_obj)
return jsonify({"response": {"code": code}}), 200
def get_results_dir_by_job_id(job_id, user_ID, checkpoint='final'):
"""
get training result by job id
:param job_id:
:param user_ID:
:param checkpoint:
:return:
"""
project = job_business.get_by_job_id(job_id).project
project_name = project.name
ownership = ownership_business.get_ownership_by_owned_item(project,
'project')
if ownership.private and ownership.user.user_ID != user_ID:
raise ValueError('Authentication failed')
user_ID = ownership.user.user_ID
result_dir = os.path.join(user_directory + user_ID + '/',
project_name + '/', job_id)
filename = '{}.hdf5'.format(checkpoint)
return result_dir, filename
def get_job(job_id):
job = job_business.get_by_job_id(job_id)
job = json_utility.convert_to_json(job.to_mongo())
return jsonify({"response": job}), 200
def resume_by_id(served_model_id, user_ID, model_name):
"""
:param served_model_id:
:param user_ID:
:param model_name:
:return:
"""
served_model = served_model_business.get_by_id(served_model_id)
job_id = served_model.jobID
job = job_business.get_by_job_id(job_id)
# if not deployed do the deployment
try:
served_model_business.get_by_job(job)
model_type = job.model.category
if model_type == ModelType['neural_network'] \
or model_type == ModelType['unstructured']:
export_path, version = model_service.export(job_id, user_ID)
else:
result_sds = staging_data_set_business.get_by_job_id(job_id)
saved_model_path_array = result_sds.saved_model_path.split('/')
version = saved_model_path_array.pop()
export_path = '/'.join(saved_model_path_array)
cwd = os.getcwd()
deploy_name = job_id + '-serving'
service_name = "my-" + job_id + "-service"
port = port_for.select_random(ports=set(range(30000, 32767)))
# export_path = "/home/root/work/user_directory" + \
# export_path.split("/user_directory", 1)[1]
export_path = "/home/root/work/user_directory" + export_path.split("/user_directory", 1)[1]
# print('resume path', export_path)
kube_json = {
"apiVersion": "apps/v1beta1",
"kind": "Deployment",
"metadata": {
"name": deploy_name
},
"spec": {
"template": {
"metadata": {
"labels": {
"app": job_id
}
},
"spec": {
"containers": [
{
"name": job_id,
"image": "10.52.14.192/gzyw/serving_app",
"imagePullPolicy": "IfNotPresent",
"ports": [{
"containerPort": 9000,
}],
"stdin": True,
"command": ['tensorflow_model_server'],
"args": ['--enable_batching',
'--port={port}'.format(
port=SERVING_PORT),
'--model_name={name}'.format(
name=model_name),
'--model_base_path={export_path}'.format(
export_path=export_path)],
"volumeMounts": [
{
"mountPath": "/home/root/work/user_directory",
"name": "nfsvol"
},
]
}
],
"volumes": [
{
"name": "nfsvol",
"persistentVolumeClaim": {
"claimName": "nfs-pvc"
}
},
]
},
},
}
}
service_json = {
"kind": "Service",
"apiVersion": "v1",
"metadata": {
"name": service_name
},
"spec": {
"type": "NodePort",
"ports": [
{
"port": 9000,
"nodePort": port
}
],
"selector": {
"app": job_id
}
}
}
api = kube_service.deployment_api
s_api = kube_service.service_api
resp = api.create_namespaced_deployment(body=kube_json,
namespace=NAMESPACE)
s_api.create_namespaced_service(body=service_json, namespace=NAMESPACE)
# 更新数据库中的port
new_server = '10.52.14.182:' + str(port)
update = {'server': new_server}
served_model_business.update_info(served_model_id, update)
return new_server
except DoesNotExist:
return False
def first_deploy(user_ID, job_id, name, description, input_info, output_info,
examples, server, input_type, model_name, projectId,
is_private,
**optional):
"""
:param user_ID:
:param job_id:
:param name:
:param description:
:param input_info:
:param output_info:
:param examples:
:param server:
:param input_type:
:param model_name:
:param is_private:
:param optional:
:return:
"""
job = job_business.get_by_job_id(job_id)
job_info = job.to_mongo()
project = project_business.get_by_id(projectId)
related_fields = project.related_fields
related_tasks = project.related_tasks
tags = project.tags
# if not deployed do the deployment
try:
served_model_business.get_by_job(job)
except DoesNotExist:
model_type = job.model.category
if model_type == ModelType['neural_network'] \
or model_type == ModelType['unstructured']:
export_path, version = model_service.export(job_id, user_ID)
else:
result_sds = staging_data_set_business.get_by_job_id(job_id)
saved_model_path_array = result_sds.saved_model_path.split('/')
version = saved_model_path_array.pop()
export_path = '/'.join(saved_model_path_array)
cwd = os.getcwd()
deploy_name = job_id + '-serving'
service_name = "my-" + job_id + "-service"
port = port_for.select_random(ports=set(range(30000, 32767)))
export_path = "/home/root/work/user_directory" + \
export_path.split("/user_directory", 1)[1]
# export_path = "/home/root/work/user_directory" + export_path.split("/user_directory", 1)[1]
kube_json = {
"apiVersion": "apps/v1beta1",
"kind": "Deployment",
"metadata": {
"name": deploy_name
},
"spec": {
"template": {
"metadata": {
"labels": {
"app": job_id
}
},
"spec": {
"containers": [
{
"name": job_id,
"image": "10.52.14.192/gzyw/serving_app",
"imagePullPolicy": "IfNotPresent",
"ports": [{
"containerPort": 9000,
}],
"stdin": True,
"command": ['tensorflow_model_server'],
"args": ['--enable_batching',
'--port={port}'.format(
port=SERVING_PORT),
'--model_name={name}'.format(
name=model_name),
'--model_base_path={export_path}'.format(
export_path=export_path)],
"volumeMounts": [
{
"mountPath": "/home/root/work/user_directory",
"name": "nfsvol"
},
]
}
],
"volumes": [
{
"name": "nfsvol",
"persistentVolumeClaim": {
"claimName": "nfs-pvc"
}
},
]
},
},
}
}
service_json = {
"kind": "Service",
"apiVersion": "v1",
"metadata": {
"name": service_name
},
"spec": {
"type": "NodePort",
"ports": [
{
"port": 9000,
"nodePort": port
}
],
"selector": {
"app": job_id
}
}
}
# import json
# from server3.utility import file_utils
# file_utils.write_to_filepath(json.dumps(kube_json), './jupyter_app.json')
# return
api = kube_service.deployment_api
s_api = kube_service.service_api
resp = api.create_namespaced_deployment(body=kube_json,
namespace=NAMESPACE)
s_api.create_namespaced_service(body=service_json, namespace=NAMESPACE)
# tf_model_server = './tensorflow_serving/model_servers/tensorflow_model_server'
# p = subprocess.Popen([
# tf_model_server,
# '--enable_batching',
# '--port={port}'.format(port=SERVING_PORT),
# '--model_name={name}'.format(name=name),
# '--model_base_path={export_path}'.format(export_path=export_path)
# ], start_new_session=True)
# add a served model entity
server = server.replace('9000', str(port))
data_fields = job_info['params']['fit']['data_fields']
job_info['staging_data_set'] = job['staging_data_set'][
'name'] if job['staging_data_set'] else None
job_info['staging_data_set_id'] = job['staging_data_set'][
'id'] if job['staging_data_set'] else None
staging_data_demo = staging_data_service.get_first_one_by_staging_data_set_id(
job_info['staging_data_set_id'])
one_input_data_demo = []
for each_feture in job_info['params']['fit']['data_fields'][0]:
one_input_data_demo.append(staging_data_demo[each_feture])
input_data_demo_string = '[' + ",".join(
str(x) for x in one_input_data_demo) + ']'
input_data_demo_string = '[' + input_data_demo_string + ',' + input_data_demo_string + ']'
return first_save_to_db(user_ID, name, description, input_info,
output_info,
examples, version,
deploy_name, server,
input_type, export_path, job,
job_id, model_name,
related_fields,
related_tasks, tags, is_private, data_fields,
input_data_demo_string,
service_name,projectId,
**optional)
