def undeploy_by_id(served_model_id):
"""
:param served_model_id:
:return:
"""
served_model = served_model_business.get_by_id(served_model_id)
kube_service.delete_deployment(served_model.deploy_name)
kube_service.delete_service(served_model.service_name)
return True
def remove_by_id(served_model_id):
# delete
served_model = served_model_business.get_by_id(served_model_id)
try:
kube_service.delete_deployment(served_model.name)
kube_service.delete_service(served_model.service_name)
# served_model_business.terminate_by_id(served_model_id)
except:
pass
served_model_business.remove_by_id(served_model_id)
def get_by_model_id(model_ID):
"""
:param category:
:return:
"""
model = served_model_business.get_by_id(model_ID)
# public_all = [each_model.to_mongo() for each_model in public_all]
model = json_utility.me_obj_list_to_json_list(
[model])
print(model)
model[0] = build_how_to_use_code(model[0])
return model
def get_all_jobs_of_project(project_id, categories, status=None):
"""
get all jobs and job info of a project
:param project_id:
:param categories:
:param status:
:return:
"""
from server3.business import job_business
# jobs = project_business.get_by_id(project_id)['jobs']
jobs = job_business.get_by_project(project_id).order_by('-create_time')
history_jobs = {c: [] for c in categories}
for job in jobs:
# keys = history_jobs.keys()
for key in categories:
if status is None:
check = job[key]
else:
check = job[key] and (job['status'] == status)
if check:
job_info = job.to_mongo()
# model/toolkit info
# job_info[key] = {
# 'item_id': job[key]['id'],
# 'name': job[key]['name'],
# 'category': job[key]['category'],
# 'parameter_spec': job[key]['parameter_spec'],
# 'steps': job[key]['steps']
# }
job_info[key] = job[key].to_mongo()
# source staging data set info
job_info['staging_data_set'] = job['staging_data_set'][
'name'] if job['staging_data_set'] else None
job_info['staging_data_set_id'] = job['staging_data_set'][
'id'] if job['staging_data_set'] else None
# result sds info
# object 用的是 .id json用 _id
if key == 'model':
try:
result_sds = staging_data_set_business.get_by_job_id(
job['id']).to_mongo()
if result_sds:
# model results
job_info['results'] = result_sds
metrics_status = [
sd.to_mongo() for sd in staging_data_business.
get_by_staging_data_set_id(
result_sds['_id']).order_by('n')
]
# metrics_status.sort(key=lambda x: x['n'])
job_info['metrics_status'] = metrics_status
if len(metrics_status) > 0:
total_steps = get_total_steps(job)
job_info['percent'] = \
metrics_status[-1]['n'] / total_steps * 100
if job_info['status'] == 200:
job_info['percent'] = 100
job_info['results_staging_data_set_id'] = \
result_sds[
'_id'] if result_sds else None
except DoesNotExist:
result_sds = None
if job['status'] == 200 and key == 'model':
temp_data_fields = job_info['params']['fit']['data_fields']
if not isinstance(temp_data_fields[0], list):
job_info['params']['fit']['data_fields'] = [
temp_data_fields
]
print(job_info['params']['fit']['data_fields'][0])
# model running status info
# if key == 'model':
# job_name = KUBE_NAME['model'].format(job_id=job['id'])
# job_info = kube_service.get_job_status(job_info, job_name)
# 获取 served_model 数据库中的信息
served_model_id = job_info.get('served_model')
if served_model_id:
served_model = served_model_business.get_by_id(
served_model_id)
# 获取 kube 中部署模型的状态
served_model = kube_service.get_deployment_status(
served_model)
served_model = served_model.to_mongo()
# 获取训练 served_model 时所用的数据的第一条
staging_data_demo = staging_data_service.get_first_one_by_staging_data_set_id(
job_info['staging_data_set_id'])
one_input_data_demo = []
for each_feture in \
job_info['params']['fit']['data_fields'][0]:
one_input_data_demo.append(
staging_data_demo[each_feture])
input_data_demo_string = '[' + ",".join(
str(x) for x in one_input_data_demo) + ']'
input_data_demo_string = '[' + input_data_demo_string + ',' + input_data_demo_string + ']'
print(input_data_demo_string)
# 生成use代码
job_info["served_model"] = served_model
job_info["served_model"][
"input_data_demo_string"] = input_data_demo_string
job_info = build_how_to_use_code(job_info)
else:
served_model = None
job_info["served_model"] = served_model
history_jobs[key].append(job_info)
break
return history_jobs
def resume_by_id(served_model_id, user_ID, model_name):
"""
:param served_model_id:
:param user_ID:
:param model_name:
:return:
"""
served_model = served_model_business.get_by_id(served_model_id)
job_id = served_model.jobID
job = job_business.get_by_job_id(job_id)
# if not deployed do the deployment
try:
served_model_business.get_by_job(job)
model_type = job.model.category
if model_type == ModelType['neural_network'] \
or model_type == ModelType['unstructured']:
export_path, version = model_service.export(job_id, user_ID)
else:
result_sds = staging_data_set_business.get_by_job_id(job_id)
saved_model_path_array = result_sds.saved_model_path.split('/')
version = saved_model_path_array.pop()
export_path = '/'.join(saved_model_path_array)
cwd = os.getcwd()
deploy_name = job_id + '-serving'
service_name = "my-" + job_id + "-service"
port = port_for.select_random(ports=set(range(30000, 32767)))
# export_path = "/home/root/work/user_directory" + \
# export_path.split("/user_directory", 1)[1]
export_path = "/home/root/work/user_directory" + export_path.split("/user_directory", 1)[1]
# print('resume path', export_path)
kube_json = {
"apiVersion": "apps/v1beta1",
"kind": "Deployment",
"metadata": {
"name": deploy_name
},
"spec": {
"template": {
"metadata": {
"labels": {
"app": job_id
}
},
"spec": {
"containers": [
{
"name": job_id,
"image": "10.52.14.192/gzyw/serving_app",
"imagePullPolicy": "IfNotPresent",
"ports": [{
"containerPort": 9000,
}],
"stdin": True,
"command": ['tensorflow_model_server'],
"args": ['--enable_batching',
'--port={port}'.format(
port=SERVING_PORT),
'--model_name={name}'.format(
name=model_name),
'--model_base_path={export_path}'.format(
export_path=export_path)],
"volumeMounts": [
{
"mountPath": "/home/root/work/user_directory",
"name": "nfsvol"
},
]
}
],
"volumes": [
{
"name": "nfsvol",
"persistentVolumeClaim": {
"claimName": "nfs-pvc"
}
},
]
},
},
}
}
service_json = {
"kind": "Service",
"apiVersion": "v1",
"metadata": {
"name": service_name
},
"spec": {
"type": "NodePort",
"ports": [
{
"port": 9000,
"nodePort": port
}
],
"selector": {
"app": job_id
}
}
}
api = kube_service.deployment_api
s_api = kube_service.service_api
resp = api.create_namespaced_deployment(body=kube_json,
namespace=NAMESPACE)
s_api.create_namespaced_service(body=service_json, namespace=NAMESPACE)
# 更新数据库中的port
new_server = '10.52.14.182:' + str(port)
update = {'server': new_server}
served_model_business.update_info(served_model_id, update)
return new_server
except DoesNotExist:
return False