async def CreateTask(request): task_request = json.loads(await request.json()) # 1 check the archive file size and judge the validation if task_request.get("archive", ""): archive_size = len(task_request["archive"]["data"]) if archive_size > Configs.MAX_SIZE: return web.Response(status=413, body={ "error": 413, "reason": "Request Entity Too Large", "description": "" }, content_type="application/json") # 2 add task to DB task_doc = DataTransformer.TaskRequest2TaskDoc(task_request) DBoperation = MongoDB.MonoDBOperation() result = await DBoperation.AddDB(task_doc) task_id = result["data"] task_doc["id"] = task_id task_doc.pop("_id") # 3 send response to client response = web.Response(status=200, body=json.dumps(task_doc), content_type="application/json") await response.prepare(request) await response.write_eof() # 4 make work dir for this task await IO.MakeWorkDir(task_id, logger) # 5 extract file if task_request.get("archive", ""): await IO.FileExtract(task_id, task_request["archive"]["type"], task_request["archive"]["data"], logger) return response
async def UpdateTaskDoc(request): task_id = request.match_info.get("task_id", "Wrong Task ID") update_doc = json.load(await request.json()) try: task_id = ObjectId(task_id) except: return web.Response(status=400, reason="Invalid Task ID") query = json.loads(request.json()) DBoperation = MongoDB.MonoDBOperation() result = await DBoperation.UpdateDBbyReplace(task_id, update_doc) return web.Response(status=result["status"], body=json.dumps(result["data"]), reason=result["error"])
async def TaskBrows(request): query = json.loads(await request.json()) DBoperation = MongoDB.MonoDBOperation() # Only support for key word query now result = await DBoperation.QueryDB( query["query"] if query.get("query", False) else {}) for index in range(len(result["data"])): result["data"][index]["id"] = str(result["data"][index]["_id"]) result["data"][index].pop("_id") return web.Response(status=result["status"], body=json.dumps( result["data"][query["skip"]:query["skip"] + query["limit"]]), content_type="application/json", reason=result["error"])
async def DeleteTask(request): task_id = request.match_info.get("task_id", "Wrong Task ID") logger.debug("launchtask task id check = {}".format(task_id)) try: ObjectId(task_id) except: logger.debug("Task ID invalid = {}".format(str(task_id))) return web.Response(status=404, body={ "error": 404, "reason": "Invalid Task ID", "description": "" }) DBoperation = MongoDB.MonoDBOperation() query_result = await DBoperation.QueryDB({"_id": ObjectId(task_id)}) if not len(query_result["data"]): return web.Response(status=404, body={ "error": 404, "description": "", "reason": "the task doesnot exist" }) task_doc = query_result["data"][0] if task_doc["status"] == "WAITING" or task_doc["status"] == "CREATING" or \ task_doc["status"] == "COMPLETED" or task_doc["status"] == "KILLED": await DBoperation.DelDB({"_id": ObjectId(task_id)}) response = web.Response(status=200, body=json.dumps({})) await response.prepare(request) await response.write_eof() await IO.RMWorkDir(task_id, logger) return response else: return web.Response( status=409, body=json.dumps( { "error": 409, "reason": "the task status is {}".format( task_doc["status"]), "description": "" }, ))
async def TaskStatusUpdate(request): task_id = request.match_info.get("task_id", "Wrong Task ID") task_status = request.match_info.get("status", "Wrong Task STATUS") try: ObjectId(task_id) except: logger.info("Invalid Task ID") return web.Response(status=400, body={ "error": 400, "reason": "Invalid Task ID", "description": "" }, content_type="application/json") # Update Task DOC ===> Finished # 2. query task doc info DBoperation = MongoDB.MonoDBOperation() query_result = await DBoperation.QueryDB({"_id": ObjectId(task_id)}) if len(query_result["data"]) == 0: return web.Response(status=404, body={ "error": 404, "reason": "Invalid Task ID", "description": "" }) # 3. update task doc info task_doc = query_result["data"][0] task_doc["status"] = task_status logger.debug("check updated task doc = {}".format(task_doc)) update_result = await DBoperation.UpdateDBbyReplace( {"_id": ObjectId(task_id)}, task_doc) logger.debug("check task status = {}".format(task_status)) # if the task is finished, should kill the task if task_status == "COMPLETED": url = Configs.Marathon_ADDRESS + "/v2/apps/{app_id}/".format( app_id="mlge1." + task_id) response = requests.delete(url=url) logger.debug("check delete response from marathon = {}".format( response.json())) return web.Response(status=200)
async def GetTaskDoc(request): task_id = request.match_info.get("task_id", "Wrong Task ID") try: task_id = ObjectId(task_id) except: logger.info("Invalid Task ID") return web.Response(status=400, body={ "error": 400, "reason": "Invalid Task ID", "description": "" }, content_type="application/json") DBoperation = MongoDB.MonoDBOperation() result = await DBoperation.QueryDB({"_id": task_id}) for index in range(len(result["data"])): result["data"][index]["id"] = str(result["data"][index]["_id"]) result["data"][index].pop("_id") return web.Response(status=result["status"], body=json.dumps(result["data"]), content_type="application/json", reason=result["error"])
async def KillTask(request): task_id = request.match_info.get("task_id", "Wrong Task ID") logger.debug("KillTask: task id check = {}".format(task_id)) try: ObjectId(task_id) except: logger.debug("Task ID invalid = {}".format(str(task_id))) return web.Response(status=404, body={ "error": 404, "reason": "Invalid Task ID", "description": "" }) DBoperation = MongoDB.MonoDBOperation() query_result = await DBoperation.QueryDB({"_id": ObjectId(task_id)}) if not len(query_result["data"]): return web.Response(status=404, body={ "error": 404, "description": "", "reason": "the task doesn't exist" }) task_doc = query_result["data"][0] # check task status if task_doc["status"] == "DEPLOYING" or task_doc["status"] == "RUNNING": # modify task status to KILLING task_doc["status"] = "KILLING" logger.debug("check updated task doc = {}".format(task_doc)) update_result = await DBoperation.UpdateDBbyReplace( {"_id": ObjectId(task_id)}, task_doc) # return response to client task_doc = update_result["data"] logger.debug("check update result = {}".format(task_doc)) task_doc["id"] = str(task_doc["_id"]) task_doc.pop("_id") response = web.Response(status=200, body=json.dumps(task_doc)) await response.prepare(request) await response.write_eof() # submit kill request to marathon marathon_request = DataTransformer.TaskDoc2MarathonRequest( task_doc, task_id) marathon_response = requests.delete( Configs.Marathon_ADDRESS + "/v2/apps/{app_id}".format(app_id="mlge1." + task_id), ) status = marathon_response.status_code marathon_response = marathon_response.json() logger.debug("kill request commit successfully") if status >= 400: # marathon cannot kill task logger.error( "marathon kill task failed , status = {} and response = {}". format(status, marathon_response)) return web.Response( status=409, body=json.dumps({ "error": 409, "reason": "marathon failure, task status: {}".format(status), "description": "" })) # update task doc query_result = await DBoperation.QueryDB({"_id": ObjectId(task_id)}) task_doc = query_result["data"][0] task_doc["status"] = "KILLED" # check task status logger.debug("check updated task doc = {}".format(task_doc)) result = await DBoperation.UpdateDBbyReplace( {"_id": ObjectId(task_id)}, task_doc) if result["status"] != 200: logger.error( "update task doc error, update doc = {}".format(task_doc)) logger.debug("task doc DB update successfully") return response else: return web.Response(status=409, body=json.dumps( { "error": 409, "reason": "status failure, task status: {}".format( task_doc["status"]), "description": "" }, ))
async def LaunchTask(request): # 1. check task id validation task_id = request.match_info.get("task_id", "Wrong Task ID") task_request = json.loads(await request.json()) logger.debug("launchtask task id check = {}".format(task_id)) try: ObjectId(task_id) except: logger.debug("Task ID invalid = {}".format(str(task_id))) return web.Response(status=404, body={ "error": 404, "reason": "Invalid Task ID", "description": "" }) # 2. query task doc info DBoperation = MongoDB.MonoDBOperation() query_result = await DBoperation.QueryDB({"_id": ObjectId(task_id)}) if len(query_result["data"]) == 0: return web.Response(status=404, body={ "error": 404, "reason": "Invalid Task ID", "description": "" }) # 3. update task doc info task_doc = query_result["data"][0] task_doc["status"] = "WAITING" logger.debug("check updated task doc = {}".format(task_doc)) update_result = await DBoperation.UpdateDBbyReplace( {"_id": ObjectId(task_id)}, task_doc) # 4. return response to client task_doc = update_result["data"] logger.debug("check update result = {}".format(task_doc)) task_doc["id"] = str(task_doc["_id"]) task_doc.pop("_id") response = web.Response(status=200, body=json.dumps(task_doc)) await response.prepare(request) await response.write_eof() # submit task to marathon marathon_request = DataTransformer.TaskDoc2MarathonRequest( task_doc, task_id) # marathon_response, status = await MarathonLayer.MarathonPost( # "http://192.168.64.57:8080/v2/apps", json_data=marathon_request) marathon_response = requests.post(Configs.Marathon_ADDRESS + "/v2/apps", json=marathon_request) status = marathon_response.status_code marathon_response = marathon_response.json() logger.debug("marathon task commit successfully") if status >= 400: logger.error( "marathon create task failed , status = {} and response = {}". format(status, marathon_response)) # update task doc update_task_doc = DataTransformer.MarathonResponse2TaskDoc( marathon_response) result = await DBoperation.UpdateDBbyReplace({"_id": ObjectId(task_id)}, update_task_doc) if result["status"] != 200: logger.error( "update task doc error, update doc = {}".format(update_task_doc)) logger.debug("task doc DB update successfully") return response