def get(self, request): task_id = request.GET.get('task_id') obj = InferTestTask.objects.get(task_id=task_id) host_id = obj.host_id version = obj.infer_version task_type = obj.infer_task_type use_model = obj.use_model token = {"token": None} token["token"] = request.META.get('HTTP_TOKEN') user_name = get_username(token) data = get_global_conf(task_type, version, types="release", user_name=user_name.username) cmd = data["cmd"] model_release.apply_async( args=[host_id, version, task_type, cmd, task_id, use_model], countdown=5) resp = JsonResponse({ "errno": RET.OK, "data": { "task_id": task_id }, "message": "success" }) return resp
def get(self, request): type_id = request.GET.get('type') version = request.GET.get('docker') try: token = {"token": None} token["token"] = request.META.get('HTTP_TOKEN') user_name = get_username(token) docker_type = str(type_id) data = get_global_conf(docker_type, version, user_name=user_name.username) if data: resp = JsonResponse({ "errno": RET.OK, "data": data, "message": "success" }) else: resp = JsonResponse({ "errno": RET.NODATA, "data": None, "message": "No Data" }) except Exception as e: import traceback logger.error(traceback.format_exc()) resp = JsonResponse({ "errno": RET.IOERR, "data": None, "message": "Data IO Error" }) return resp
def get(self, request): task_id = request.GET.get('task_id') obj = InferTestTask.objects.get(task_id=task_id) host_id = obj.host_id version = obj.infer_version gpu_id = obj.gpu_id task_type = obj.infer_task_type use_model = obj.use_model token = {"token": None} token["token"] = request.META.get('HTTP_TOKEN') user_name = get_username(token) data = get_global_conf(task_type, version, types="tensorRt", user_name=user_name.username) cmd = data["cmd"] conf_path = os.path.join(TMP_PATH, task_id) model_tensorRt.apply_async(args=[ host_id, version, task_type, gpu_id, cmd, task_id, conf_path, use_model ], countdown=5) # model_tensorRt(host_id, version, task_type, gpu_id, cmd, task_id, conf_path, use_model) resp = JsonResponse({ "errno": RET.OK, "data": { "task_id": task_id }, "message": "success" }) return resp
def get(self, request): task_id = request.GET.get('task_id') obj = DataProHandle.objects.get(task_id=task_id) task_type = obj.data_task_type data_name = obj.src_dir_name task_name = obj.dir_name version = obj.data_version host = obj.data_host_id token = {"token": None} token["token"] = request.META.get('HTTP_TOKEN') user_name = get_username(token) data = get_global_conf(task_type, version, types="prodata", user_name=user_name.username) cmd = data["cmd"] conf_path = os.path.join(TMP_PATH, task_id) input_path = os.path.join(SRC_DATA, task_type, data_name) output_path = os.path.join(TRAIN_DATA, task_name) docker = DOCKER_URL + "{}:{}".format(task_type, version) d = op_tar("241", os.path.join(SRC_DATA, task_type), data_name) if not d: rest = download( host, os.path.join(SRC_DATA, task_type, data_name + ".tar.gz"), os.path.join(SRC_DATA, task_type)) if not rest: dest_scp, dest_sftp, dest_ssh = client(ids="241", types="docker") cmd_1 = "rm -r {}".format( os.path.join(SRC_DATA, task_type, data_name + ".tar.gz")) stdin, stdout, stderr = dest_ssh.exec_command(cmd_1) print(stderr.read()) s = pro_data.apply_async(args=[ host, docker, cmd, input_path, output_path, conf_path, task_id ], countdown=5) resp = JsonResponse({ "errno": RET.OK, "data": None, "message": "success" }) else: resp = JsonResponse({ "errno": RET.IOERR, "data": None, "message": "Failed" }) else: resp = JsonResponse({ "errno": RET.IOERR, "data": None, "message": "Failed" }) return resp
def get(self, request): task_id = request.GET.get('task_id') obj = InferTestTask.objects.get(task_id=task_id) host_id = obj.host_id version = obj.infer_version gpu_id = obj.gpu_id task_type = obj.infer_task_type use_model = obj.use_model seg_tag = obj.seg_tag image_type = obj.image_type full_size = image_type_map[image_type] token = {"token": None} token["token"] = request.META.get('HTTP_TOKEN') user_name = get_username(token) src_path = os.path.join( os.path.abspath(os.path.join(os.getcwd(), 'demo', task_type))) dest_path = os.path.join(TEST_DATA, task_type, task_id) dest_scp, dest_sftp, dest_ssh = client(ids=host_id, types="docker") try: dest_sftp.stat(dest_path) except IOError: cmd = "mkdir -p {}".format(dest_path) stdin1, stdout1, stderr1 = dest_ssh.exec_command(cmd) print(stderr1.read()) dest_scp.put(src_path, dest_path, recursive=True) cmd_2 = "cd {} && mv {}/* . && rm -r {}".format( dest_path, os.path.join(dest_path, task_type), task_type) stdin2, stdout2, stderr2 = dest_ssh.exec_command(cmd_2) print(stderr2.read()) dest_scp.close() dest_sftp.close() dest_ssh.close() data = get_global_conf(task_type, version, types="consistency", user_name=user_name.username) cmd = data["cmd"] conf_map = data["conf_common"] conf_path = os.path.join(TMP_PATH, task_id) # model_con.apply_async( # args=[host_id, version, task_type, gpu_id, dest_path, cmd, task_id, conf_path,use_model, conf_map, seg_tag, full_size], # countdown=5) model_con(host_id, version, task_type, gpu_id, dest_path, cmd, task_id, conf_path, use_model, conf_map, seg_tag, full_size) resp = JsonResponse({ "errno": RET.OK, "data": { "task_id": task_id }, "message": "success" }) return resp
def get(self, request): task_id = request.GET.get('task_id') page = int(request.GET.get('page')) token = {"token": None} token["token"] = request.META.get('HTTP_TOKEN') user_name = get_username(token) try: obj = TrainTask.objects.get(task_id=task_id) task_type = obj.task_type version = obj.version host = obj.host_id except Exception as e: import traceback logger.error(traceback.format_exc()) resp = JsonResponse({ "errno": RET.DBERR, "data": None, "message": "Failed" }) else: data = get_global_conf(task_type, version, types="train", host=host, user_name=user_name.username) if page <= len(data["conf"]): conf_path = data["conf"][page - 1] conf_name = os.path.basename(conf_path) dest_path = os.path.abspath( os.path.join(os.getcwd(), 'tmp', task_id)) src_path = os.path.join(TMP_PATH, task_id) conf_data = op_conf(src_path=src_path, docker_type=task_type, docker_version=version, conf_path=conf_path, types=1, dest_path=dest_path) resp = JsonResponse({ "errno": RET.OK, "data": { "data": conf_data, "conf_name": conf_name }, "message": "success" }) else: resp = JsonResponse({ "errno": RET.IOERR, "data": None, "message": "Failed" }) return resp
def post(self, request): task_id = "InferTestTask_" + str(int(time.time())) req_dict = json.loads(request.read()) version = req_dict["docker_tag"] task_type = req_dict["task_type"] data_name = req_dict["data_name"] use_model = req_dict["models"] types = req_dict["types"] gpu = req_dict["gpu"] host_id = "165" try: obj = InferTestTask() obj.task_id = task_id obj.start_time = time.strftime("%Y-%m-%d %H:%M", time.localtime()) obj.host_id = host_id obj.gpu_id = gpu obj.use_model = json.dumps(use_model) obj.infer_task_type = task_type obj.infer_version = version if int(types) == 2: data_name = "test_data" obj.data_name = data_name obj.types = types obj.status = "progress" obj.save() token = {"token": None} token["token"] = request.META.get('HTTP_TOKEN') user_name = get_username(token) data = get_global_conf(task_type, version, types="test", host=host_id, user_name=user_name.username) conf_count = len(data["conf"]) except Exception as e: import traceback logger.error(traceback.format_exc()) resp = JsonResponse({ "errno": RET.DBERR, "data": None, "message": "DB Error" }) else: resp = JsonResponse({ "errno": RET.OK, "data": { "total": conf_count, "task_id": task_id }, "message": "success" }) return resp
def get(self, request): task_id = request.GET.get('task_id') try: obj = InferTestTask.objects.get(task_id=task_id) task_type = obj.infer_task_type version = obj.infer_version use_model = obj.use_model data_name = obj.data_name types = obj.types gpu_id = obj.gpu_id host = obj.host_id token = {"token": None} token["token"] = request.META.get('HTTP_TOKEN') user_name = get_username(token) except Exception as e: import traceback logger.error(traceback.format_exc()) resp = JsonResponse({ "errno": RET.DBERR, "data": None, "message": "Failed" }) else: src_path = model_pro_data(host, task_id, json.loads(use_model), types, task_type, data_name) data = get_global_conf(task_type, version, types="test", user_name=user_name.username) cmd = data["cmd"] conf_map = data["conf_common"] # conf_map = "" conf_path = os.path.join(TMP_PATH, task_id) model_eval.apply_async(args=[ host, version, task_type, gpu_id, src_path, cmd, task_id, conf_path, types, use_model, conf_map ], countdown=5) # model_eval(host, version, task_type, gpu_id, src_path, cmd, task_id, conf_path, types, use_model, conf_map) resp = JsonResponse({ "errno": RET.OK, "data": { "task_id": task_id }, "message": "success" }) return resp
def post(self, request): req_dict = json.loads(request.read()) task_id = req_dict["task_id"] op_type = req_dict["type"] token = {"token": None} token["token"] = request.META.get('HTTP_TOKEN') user_name = get_username(token) try: obj = TrainTask.objects.get(task_id=task_id) pro_id = obj.pro_id task_name = obj.task_name task_type = obj.task_type data_name = obj.train_data_name version = obj.version host_id = obj.host_id gpu_id = obj.gpu_id weight = obj.weight status = obj.status if int(op_type) == 1: if status in ["progress"]: task = run_train.AsyncResult(pro_id) if task: if task.state == 'PENDING': celery_app.control.revoke(pro_id, terminate=True) dest_scp, dest_sftp, dest_ssh = client(ids=host_id) cmd_1 = "docker ps -a | grep {}".format(task_id) stdin1, stdout1, stderr1 = dest_ssh.exec_command( cmd_1) reslut = stdout1.read() if reslut: cmd_2 = "docker stop {}".format(task_id) stdin2, stdout2, stderr2 = dest_ssh.exec_command( cmd_2) dest_scp.close() dest_sftp.close() dest_ssh.close() obj.status = "stop" obj.save() resp = JsonResponse({ "errno": RET.OK, "data": None, "message": "task stop" }) else: resp = JsonResponse({ "errno": RET.NODATA, "data": None, "message": "task no exist or success" }) else: resp = JsonResponse({ "errno": RET.NODATA, "data": None, "message": "task no exist or success" }) elif int(op_type) == 2: if status in ["stop", "failed"]: model_out = TRAIN_OUTPUT_PATH.format(task_name) input_name = os.path.join(TRAIN_DATA, data_name.dir_name) output_name = os.path.join(TRAIN_PATH, task_name) data = get_global_conf(task_type, version, types="train", user_name=user_name.username) cmd = data["cmd"] docker = DOCKER_URL + "{}:{}".format(task_type, version) dest_scp, dest_sftp, dest_ssh = client(ids=host_id) li = dest_sftp.listdir(model_out) if len(li) >= 2: weight_path = model_out else: weight_path = os.path.join( TRAIN_MODEL_PATH.format(task_name), weight) cmd_1 = "docker ps -a | grep {}".format(task_id) stdin1, stdout1, stderr1 = dest_ssh.exec_command(cmd_1) if stdout1.read(): cmd_2 = "docker rm {}".format(task_id) stdin2, stdout2, stderr2 = dest_ssh.exec_command(cmd_2) run_task = run_train.apply_async(args=[ task_name, docker, host_id, gpu_id, weight_path, task_id, input_name, output_name, cmd ], countdown=5) obj.status = "progress" obj.pro_id = run_task.id obj.save() dest_scp.close() dest_sftp.close() dest_ssh.close() resp = JsonResponse({ "errno": RET.OK, "data": None, "message": "success" }) else: resp = JsonResponse({ "errno": RET.DATAEXIST, "data": None, "message": "Task in progress or completed" }) else: resp = JsonResponse({ "errno": RET.PARAMERR, "data": None, "message": "params error" }) except Exception as e: import traceback logger.error(traceback.format_exc()) resp = JsonResponse({ "errno": RET.DBERR, "data": None, "message": "DB Error" }) return resp
def get(self, request): task_id = request.GET.get('task_id') obj = TrainTask.objects.get(task_id=task_id) task_name = obj.task_name task_type = obj.task_type data_name = obj.train_data_name version = obj.version host_id = obj.host_id gpu_id = obj.gpu_id weight = obj.weight if not obj.status: token = {"token": None} token["token"] = request.META.get('HTTP_TOKEN') user_name = get_username(token) input_name = os.path.join(TRAIN_DATA, data_name.dir_name) output_name = os.path.join(TRAIN_PATH, task_name) data = get_global_conf(task_type, version, types="train", user_name=user_name.username) cmd = data["cmd"] docker = DOCKER_URL + "{}:{}".format(task_type, version) weight_path = "" if weight: weight_path = os.path.join(TRAIN_MODEL_PATH.format(task_name), weight) run_task = run_train.apply_async(args=[ task_name, docker, host_id, gpu_id, weight_path, task_id, input_name, output_name, cmd ], countdown=5) # run_train( # task_name, docker, host_id, gpu_id, weight_path, task_id, input_name, output_name, cmd) obj.status = "progress" obj.pro_id = run_task.id obj.save() u_obj = UserInfo.objects.get(username=user_name) if u_obj.default_docker: d = json.loads(u_obj.default_docker) if task_type in d: if d[task_type] != version: d[task_type] = version else: d[task_type] = version else: d = {} d[task_type] = version u_obj.default_docker = json.dumps(d) u_obj.save() resp = JsonResponse({ "errno": RET.OK, "data": None, "message": "success" }) else: resp = JsonResponse({ "errno": RET.DATAEXIST, "data": None, "message": "The task is already start" }) return resp
def post(self, request): task_id = "TrainTask_" + str(int(time.time())) req_dict = json.loads(request.read()) type_id = req_dict["task_type"] old_id = req_dict["task_id"] version = req_dict["docker_tag"] data_name = req_dict["data_name"] comment = req_dict["comment"] task_name = req_dict["task_name"] gpu = req_dict["gpu"] weight = req_dict["weight"] task_type = str(type_id) host = "165" if old_id: obj = TrainTask.objects.get(task_id=old_id) old_task_name = obj.task_name else: old_task_name = "" if TrainTask.objects.filter( Q(task_type=task_type) & Q(task_name=task_name)).count() > 0: resp = JsonResponse({ "errno": RET.DATAEXIST, "data": None, "message": "Task Existed" }) else: make_folder(task_name, host) if weight: init_weight(host, old_task_name, task_name, weight, task_type) train_pro_data(host, data_name) token = {"token": None} token["token"] = request.META.get('HTTP_TOKEN') user_name = get_username(token) data = get_global_conf(task_type, version, types="train", user_name=user_name.username) conf_count = len(data["conf"]) try: obj = DataProHandle.objects.get(dir_name=data_name) train_task = TrainTask() train_task.task_id = task_id train_task.task_type = task_type train_task.start_time = time.strftime("%Y-%m-%d %H:%M", time.localtime()) train_task.host_id = host train_task.gpu_id = gpu train_task.task_name = task_name train_task.train_data_name = obj train_task.infos = comment train_task.version = version train_task.weight = weight train_task.save() except Exception as e: import traceback logger.error(traceback.format_exc()) resp = JsonResponse({ "errno": RET.DBERR, "data": None, "message": "DB Error" }) else: resp = JsonResponse({ "errno": RET.OK, "data": { "total": conf_count, "task_id": task_id }, "message": "success" }) return resp
def post(self, request): req_dict = json.loads(request.read()) task_id = "DataProHandle_" + str(int(time.time())) type_id = req_dict["task_type"] old_id = req_dict["task_id"] version = req_dict["docker_tag"] data_name = req_dict["data_name"] comment = req_dict["comment"] task_name = req_dict["task_name"] task_type = str(type_id) # conf_json = req_dict["conf_json"] host = "165" if DataProHandle.objects.filter( Q(data_task_type=task_type) & Q(dir_name=task_name)).count() > 0: resp = JsonResponse({ "errno": RET.DATAEXIST, "data": None, "message": "Data Existed" }) else: try: obj = DataProHandle() obj.task_id = task_id obj.start_time = time.strftime("%Y-%m-%d %H:%M", time.localtime()) obj.dir_name = task_name obj.data_task_type = task_type obj.data_version = version obj.src_dir_name = data_name obj.data_host_id = host obj.infos = comment obj.status = "progress" obj.save() token = {"token": None} token["token"] = request.META.get('HTTP_TOKEN') user_name = get_username(token) data = get_global_conf(task_type, version, types="prodata", host=host, user_name=user_name.username) conf_count = len(data["conf"]) except Exception as e: import traceback logger.error(traceback.format_exc()) resp = JsonResponse({ "errno": RET.DBERR, "data": None, "message": "Failed" }) else: resp = JsonResponse({ "errno": RET.OK, "data": { "total": conf_count, "task_id": task_id }, "message": "success" }) return resp
def post(self, request): task_id = "Release_" + str(int(time.time())) req_dict = json.loads(request.read()) version = req_dict["docker_tag"] task_type = req_dict["task_type"] use_model = req_dict["models"] if use_model: if len(use_model) > 1 or len(list(use_model.values())[0]) > 1: resp = JsonResponse({ "errno": RET.NODATA, "data": None, "message": "Only one model can be selected" }) else: for k, v in use_model.items(): obj = TrainTask.objects.get(task_name=k) host_id = obj.host_id if obj.status == "completed": resp = JsonResponse({ "errno": RET.NODATA, "data": None, "message": "The model has been released" }) else: try: obj = InferTestTask() obj.task_id = task_id obj.start_time = time.strftime("%Y-%m-%d %H:%M", time.localtime()) obj.host_id = host_id obj.use_model = json.dumps(use_model) obj.infer_task_type = task_type obj.infer_version = version obj.types = "5" obj.status = "progress" obj.save() token = {"token": None} token["token"] = request.META.get('HTTP_TOKEN') user_name = get_username(token) data = get_global_conf(task_type, version, types="release", host=host_id, user_name=user_name.username) conf_count = len(data["conf"]) except Exception as e: import traceback logger.error(traceback.format_exc()) resp = JsonResponse({ "errno": RET.DBERR, "data": None, "message": "DB Error" }) else: resp = JsonResponse({ "errno": RET.OK, "data": { "total": conf_count, "task_id": task_id }, "message": "success" }) return resp else: resp = JsonResponse({ "errno": RET.NODATA, "data": None, "message": "Please select the model first" }) return resp