def main(unused_argv): job_id = FLAGS.job_id if job_id == "59ae047e0c11f35fafebc422": raise ValueError('no job_id flag') job = job_business.get_by_job_id(job_id) # project id project_id = job.project.id project = project_business.get_by_id(project_id) ow = ownership_business.get_ownership_by_owned_item(project, 'project') # user ID user_ID = ow.user.user_ID args = job.run_args try: run_model(args['conf'], args['project_id'], args['data_source_id'], args['model_id'], job_id, **args['kwargs']) except Exception: # if error send error, save error and raise error exc_type, exc_value, exc_traceback = sys.exc_info() message = { 'error': repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), 'type': 'model' } print(message) emit_error(message, str(project_id), job_id=job_id, user_ID=user_ID) save_job_status(job, error=message, status=300) else: message = { 'project_name': project.name, 'type': 'model', 'complete': True, 'content': 'Model job completed in project ' + project.name } emit_success(message, str(project_id), job_id=job_id, user_ID=user_ID)
def save_as_result(job_id, new_sds_name): job_obj = job_business.get_by_job_id(job_id) result = job_obj.result toolkit = job_obj.toolkit project_obj = job_obj.project sds_id = staging_data_set_business.add( name=new_sds_name, description='des', project=project_obj, # job=job_obj ) # 拿到原表 old_sds = StepBusiness.get_datasource(job_obj.steps) table = staging_data_business.get_by_staging_data_set_id(old_sds) table_dict = [] for i in range(len(table)): row = table[i].to_mongo().to_dict() row.pop("_id") row.pop("staging_data_set") table_dict.append(row) # 复制原表 staging_data_business.add_many(staging_data_set=sds_id, data_array=table_dict) # 保存结果 save_result_sub(result, sds_id, toolkit)
def save_result(job_id): job_obj = job_business.get_by_job_id(job_id) result = job_obj.result toolkit = job_obj.toolkit sds_id = StepBusiness.get_datasource(job_obj.steps) save_result_sub(result, sds_id, toolkit)
def run_job(): data = request.get_json() job_id = data['section_id'] project_id = data["project_id"] job_obj = job_business.get_by_job_id(job_id) project = project_business.get_by_id(project_id) ow = ownership_business.get_ownership_by_owned_item(project, 'project') # user ID user_ID = ow.user.user_ID type = None try: if job_obj.toolkit: type = 'toolkit' complete = True content = 'Toolkit job completed in project ' + project.name result = job_service.run_toolkit_job(project_id=project_id, job_obj=job_obj) elif job_obj.model: type = 'model' complete = False content = 'Model job successfully created in project ' + \ project.name result = job_service.run_model_job(project_id=project_id, job_obj=job_obj) else: return jsonify({"response": 'no model and toolkit in job object'}), 400 result = json_utility.convert_to_json(result) except Exception as e: # if error send error, save error and raise error exc_type, exc_value, exc_traceback = sys.exc_info() message = { 'error': repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), 'type': type } print(message) emit_error(message, str(project_id), job_id=job_id, user_ID=user_ID) save_job_status(job_obj, error=message, status=300) raise e # return jsonify({ # "response": { # "result": message # }}), 200 else: message = { 'project_name': project.name, 'type': type, 'complete': complete, 'content': content } emit_success(message, str(project_id), job_id=job_id, user_ID=user_ID) return jsonify({"response": {"result": result}}), 200
def wrapper(*args, **kw): # create a job # model_obj = model_business.get_by_model_id(model_id) result_dir = kwargs.get('result_dir') project_obj = project_business.get_by_id(project_id) job_obj = job_business.get_by_job_id(job_id) # update a project project_business.insert_job_by_id(project_id, job_obj.id) project_business.update_items_to_list_field(project_id, related_tasks=TYPE.get( model_obj.category, [])) # create result sds for model sds_name = '%s_%s_result' % (model_obj['name'], job_obj['id']) try: sds = staging_data_set_business.get_by_job_id(job_obj.id) except DoesNotExist: print('free to create sds') else: staging_data_set_business.remove_by_id(sds.id) finally: result_sds_obj = staging_data_set_business.add(sds_name, 'des', project_obj, job=job_obj, type='result') # run if result_dir: # result_dir += str(job_obj['id']) + '/' try: os.makedirs(result_dir) except FileExistsError: print('dir exists, no need to create') kw['result_dir'] = result_dir # generate_job_py(func, *args, **kw, result_sds=result_sds_obj, # project_id=project_id) func_result = func(*args, **kw, result_sds=result_sds_obj, project_id=project_id, job_id=job_id) # update a job job_business.end_job(job_obj) if isinstance(func_result, dict): func_result['job_id'] = str(job_obj['id']) return func_result
def to_code(): data = request.get_json() job_id = data['section_id'] project_id = data["project_id"] job_obj = job_business.get_by_job_id(job_id) project = project_business.get_by_id(project_id) ow = ownership_business.get_ownership_by_owned_item(project, 'project') # user ID user_ID = ow.user.user_ID code = job_service.model_job_to_code(project_id=project_id, job_obj=job_obj) return jsonify({"response": {"code": code}}), 200
def get_results_dir_by_job_id(job_id, user_ID, checkpoint='final'): """ get training result by job id :param job_id: :param user_ID: :param checkpoint: :return: """ project = job_business.get_by_job_id(job_id).project project_name = project.name ownership = ownership_business.get_ownership_by_owned_item(project, 'project') if ownership.private and ownership.user.user_ID != user_ID: raise ValueError('Authentication failed') user_ID = ownership.user.user_ID result_dir = os.path.join(user_directory + user_ID + '/', project_name + '/', job_id) filename = '{}.hdf5'.format(checkpoint) return result_dir, filename
def get_job(job_id): job = job_business.get_by_job_id(job_id) job = json_utility.convert_to_json(job.to_mongo()) return jsonify({"response": job}), 200
def resume_by_id(served_model_id, user_ID, model_name): """ :param served_model_id: :param user_ID: :param model_name: :return: """ served_model = served_model_business.get_by_id(served_model_id) job_id = served_model.jobID job = job_business.get_by_job_id(job_id) # if not deployed do the deployment try: served_model_business.get_by_job(job) model_type = job.model.category if model_type == ModelType['neural_network'] \ or model_type == ModelType['unstructured']: export_path, version = model_service.export(job_id, user_ID) else: result_sds = staging_data_set_business.get_by_job_id(job_id) saved_model_path_array = result_sds.saved_model_path.split('/') version = saved_model_path_array.pop() export_path = '/'.join(saved_model_path_array) cwd = os.getcwd() deploy_name = job_id + '-serving' service_name = "my-" + job_id + "-service" port = port_for.select_random(ports=set(range(30000, 32767))) # export_path = "/home/root/work/user_directory" + \ # export_path.split("/user_directory", 1)[1] export_path = "/home/root/work/user_directory" + export_path.split("/user_directory", 1)[1] # print('resume path', export_path) kube_json = { "apiVersion": "apps/v1beta1", "kind": "Deployment", "metadata": { "name": deploy_name }, "spec": { "template": { "metadata": { "labels": { "app": job_id } }, "spec": { "containers": [ { "name": job_id, "image": "10.52.14.192/gzyw/serving_app", "imagePullPolicy": "IfNotPresent", "ports": [{ "containerPort": 9000, }], "stdin": True, "command": ['tensorflow_model_server'], "args": ['--enable_batching', '--port={port}'.format( port=SERVING_PORT), '--model_name={name}'.format( name=model_name), '--model_base_path={export_path}'.format( export_path=export_path)], "volumeMounts": [ { "mountPath": "/home/root/work/user_directory", "name": "nfsvol" }, ] } ], "volumes": [ { "name": "nfsvol", "persistentVolumeClaim": { "claimName": "nfs-pvc" } }, ] }, }, } } service_json = { "kind": "Service", "apiVersion": "v1", "metadata": { "name": service_name }, "spec": { "type": "NodePort", "ports": [ { "port": 9000, "nodePort": port } ], "selector": { "app": job_id } } } api = kube_service.deployment_api s_api = kube_service.service_api resp = api.create_namespaced_deployment(body=kube_json, namespace=NAMESPACE) s_api.create_namespaced_service(body=service_json, namespace=NAMESPACE) # 更新数据库中的port new_server = '10.52.14.182:' + str(port) update = {'server': new_server} served_model_business.update_info(served_model_id, update) return new_server except DoesNotExist: return False
def first_deploy(user_ID, job_id, name, description, input_info, output_info, examples, server, input_type, model_name, projectId, is_private, **optional): """ :param user_ID: :param job_id: :param name: :param description: :param input_info: :param output_info: :param examples: :param server: :param input_type: :param model_name: :param is_private: :param optional: :return: """ job = job_business.get_by_job_id(job_id) job_info = job.to_mongo() project = project_business.get_by_id(projectId) related_fields = project.related_fields related_tasks = project.related_tasks tags = project.tags # if not deployed do the deployment try: served_model_business.get_by_job(job) except DoesNotExist: model_type = job.model.category if model_type == ModelType['neural_network'] \ or model_type == ModelType['unstructured']: export_path, version = model_service.export(job_id, user_ID) else: result_sds = staging_data_set_business.get_by_job_id(job_id) saved_model_path_array = result_sds.saved_model_path.split('/') version = saved_model_path_array.pop() export_path = '/'.join(saved_model_path_array) cwd = os.getcwd() deploy_name = job_id + '-serving' service_name = "my-" + job_id + "-service" port = port_for.select_random(ports=set(range(30000, 32767))) export_path = "/home/root/work/user_directory" + \ export_path.split("/user_directory", 1)[1] # export_path = "/home/root/work/user_directory" + export_path.split("/user_directory", 1)[1] kube_json = { "apiVersion": "apps/v1beta1", "kind": "Deployment", "metadata": { "name": deploy_name }, "spec": { "template": { "metadata": { "labels": { "app": job_id } }, "spec": { "containers": [ { "name": job_id, "image": "10.52.14.192/gzyw/serving_app", "imagePullPolicy": "IfNotPresent", "ports": [{ "containerPort": 9000, }], "stdin": True, "command": ['tensorflow_model_server'], "args": ['--enable_batching', '--port={port}'.format( port=SERVING_PORT), '--model_name={name}'.format( name=model_name), '--model_base_path={export_path}'.format( export_path=export_path)], "volumeMounts": [ { "mountPath": "/home/root/work/user_directory", "name": "nfsvol" }, ] } ], "volumes": [ { "name": "nfsvol", "persistentVolumeClaim": { "claimName": "nfs-pvc" } }, ] }, }, } } service_json = { "kind": "Service", "apiVersion": "v1", "metadata": { "name": service_name }, "spec": { "type": "NodePort", "ports": [ { "port": 9000, "nodePort": port } ], "selector": { "app": job_id } } } # import json # from server3.utility import file_utils # file_utils.write_to_filepath(json.dumps(kube_json), './jupyter_app.json') # return api = kube_service.deployment_api s_api = kube_service.service_api resp = api.create_namespaced_deployment(body=kube_json, namespace=NAMESPACE) s_api.create_namespaced_service(body=service_json, namespace=NAMESPACE) # tf_model_server = './tensorflow_serving/model_servers/tensorflow_model_server' # p = subprocess.Popen([ # tf_model_server, # '--enable_batching', # '--port={port}'.format(port=SERVING_PORT), # '--model_name={name}'.format(name=name), # '--model_base_path={export_path}'.format(export_path=export_path) # ], start_new_session=True) # add a served model entity server = server.replace('9000', str(port)) data_fields = job_info['params']['fit']['data_fields'] job_info['staging_data_set'] = job['staging_data_set'][ 'name'] if job['staging_data_set'] else None job_info['staging_data_set_id'] = job['staging_data_set'][ 'id'] if job['staging_data_set'] else None staging_data_demo = staging_data_service.get_first_one_by_staging_data_set_id( job_info['staging_data_set_id']) one_input_data_demo = [] for each_feture in job_info['params']['fit']['data_fields'][0]: one_input_data_demo.append(staging_data_demo[each_feture]) input_data_demo_string = '[' + ",".join( str(x) for x in one_input_data_demo) + ']' input_data_demo_string = '[' + input_data_demo_string + ',' + input_data_demo_string + ']' return first_save_to_db(user_ID, name, description, input_info, output_info, examples, version, deploy_name, server, input_type, export_path, job, job_id, model_name, related_fields, related_tasks, tags, is_private, data_fields, input_data_demo_string, service_name,projectId, **optional)