def get_all_jobs_of_project(project_id, categories, status=None):
"""
get all jobs and job info of a project
:param project_id:
:param categories:
:param status:
:return:
"""
from server3.business import job_business
# jobs = project_business.get_by_id(project_id)['jobs']
jobs = job_business.get_by_project(project_id).order_by('-create_time')
history_jobs = {c: [] for c in categories}
for job in jobs:
# keys = history_jobs.keys()
for key in categories:
if status is None:
check = job[key]
else:
check = job[key] and (job['status'] == status)
if check:
job_info = job.to_mongo()
# model/toolkit info
# job_info[key] = {
# 'item_id': job[key]['id'],
# 'name': job[key]['name'],
# 'category': job[key]['category'],
# 'parameter_spec': job[key]['parameter_spec'],
# 'steps': job[key]['steps']
# }
job_info[key] = job[key].to_mongo()
# source staging data set info
job_info['staging_data_set'] = job['staging_data_set'][
'name'] if job['staging_data_set'] else None
job_info['staging_data_set_id'] = job['staging_data_set'][
'id'] if job['staging_data_set'] else None
# result sds info
# object 用的是 .id json用 _id
if key == 'model':
try:
result_sds = staging_data_set_business.get_by_job_id(
job['id']).to_mongo()
if result_sds:
# model results
job_info['results'] = result_sds
metrics_status = [
sd.to_mongo() for sd in staging_data_business.
get_by_staging_data_set_id(
result_sds['_id']).order_by('n')
]
# metrics_status.sort(key=lambda x: x['n'])
job_info['metrics_status'] = metrics_status
if len(metrics_status) > 0:
total_steps = get_total_steps(job)
job_info['percent'] = \
metrics_status[-1]['n'] / total_steps * 100
if job_info['status'] == 200:
job_info['percent'] = 100
job_info['results_staging_data_set_id'] = \
result_sds[
'_id'] if result_sds else None
except DoesNotExist:
result_sds = None
if job['status'] == 200 and key == 'model':
temp_data_fields = job_info['params']['fit']['data_fields']
if not isinstance(temp_data_fields[0], list):
job_info['params']['fit']['data_fields'] = [
temp_data_fields
]
print(job_info['params']['fit']['data_fields'][0])
# model running status info
# if key == 'model':
# job_name = KUBE_NAME['model'].format(job_id=job['id'])
# job_info = kube_service.get_job_status(job_info, job_name)
# 获取 served_model 数据库中的信息
served_model_id = job_info.get('served_model')
if served_model_id:
served_model = served_model_business.get_by_id(
served_model_id)
# 获取 kube 中部署模型的状态
served_model = kube_service.get_deployment_status(
served_model)
served_model = served_model.to_mongo()
# 获取训练 served_model 时所用的数据的第一条
staging_data_demo = staging_data_service.get_first_one_by_staging_data_set_id(
job_info['staging_data_set_id'])
one_input_data_demo = []
for each_feture in \
job_info['params']['fit']['data_fields'][0]:
one_input_data_demo.append(
staging_data_demo[each_feture])
input_data_demo_string = '[' + ",".join(
str(x) for x in one_input_data_demo) + ']'
input_data_demo_string = '[' + input_data_demo_string + ',' + input_data_demo_string + ']'
print(input_data_demo_string)
# 生成use代码
job_info["served_model"] = served_model
job_info["served_model"][
"input_data_demo_string"] = input_data_demo_string
job_info = build_how_to_use_code(job_info)
else:
served_model = None
job_info["served_model"] = served_model
history_jobs[key].append(job_info)
break
return history_jobs
def list_by_project_id(project_id):
project = project_business.get_by_id(project_id)
return job_business.get_by_project(project)