def post(self): json_data = request.get_json(force=True) if json_data is None: abort_message("can not parse post data as json!") uid = json_data.get("user_uuid", None) did = json_data.get("dataset_uuid", None) task_type = json_data.get("task_type", None) if uid is None or did is None or task_type is None: abort_message( "can not find user_uuid or dataset_uuid or task_type") db_config = GLOBAL_CONFIG[DB_CONFIG_KEY].copy() task_config = config.CLASSIFY_TASK_CONFIG.copy() task_config["user_uuid"] = uid task_config["dataset_uuid"] = did task_config["model_type"] = task_type global_task_center_config = GLOBAL_CONFIG[TASK_CONFIG_KEY] merged_config = config.AnnotatorConfig(task_config, global_task_center_config) st_cmd = LatestStatusCmd(db_config, merged_config) ret_data = st_cmd() return jsonify({ "status": ret_data[0], "end_time": ret_data[1], "task_id": ret_data[2] })
def post(self): json_data = request.get_json(force=True) if json_data is None: abort_message("can not parse post data as json!") model_type = json_data.get("task_type", None) uid = json_data.get("user_uuid", None) did = json_data.get("dataset_uuid", None) data = json_data.get("data", None) if model_type is None or uid is None or did is None or data is None: abort_message("can not find uid or did or task type or data!") db_config = GLOBAL_CONFIG[DB_CONFIG_KEY].copy() # TODO more task type will add later task_config = config.CLASSIFY_TASK_CONFIG.copy() task_config["user_uuid"] = uid task_config["dataset_uuid"] = did task_config["model_type"] = model_type task_config["data"] = data global_task_center_config = GLOBAL_CONFIG[TASK_CONFIG_KEY] merged_config = config.AnnotatorConfig(task_config, global_task_center_config) merged_config["model_path"] = merged_config.get_save_path_prefix() bnpc = BatchNoDbPredictCmd(db_config, merged_config) # format = {txt:XXX, result: {label : XXX, con: XXX}result result = bnpc() return jsonify(result)
def abs_test_status(): db_config = {"database_hostname":"localhost", "database_port" : 27017, "database_type": "mongodb", "database_name": "chinese_annotator", "user_name":"anno_admin", "password": "******"} global_config, task_config = create_pred_cfgs() merged_config = config.AnnotatorConfig(task_config, global_config) st_cmd = StatusCmd(db_config, merged_config) print(st_cmd())
def abc_test_batch_predict(): db_config = {"database_hostname":"localhost", "database_port" : 27017, "database_type": "mongodb", "database_name": "chinese_annotator", "user_name":"anno_admin", "password": "******"} global_config, task_config = create_pred_cfgs() merged_config = config.AnnotatorConfig(task_config, global_config) merged_config["model_path"] = merged_config.get_save_path_prefix() # TM = TaskManager(global_config["max_process_number"], global_config["max_task_in_queue"]) btc = BatchPredictCmd(db_config, merged_config) preds = btc() print(preds)
def abc_test_batch_train(): db_config = {"database_hostname":"localhost", "database_port" : 27017, "database_type": "mongodb", "database_name": "chinese_annotator", "user_name":"anno_admin", "password": "******"} global_config, task_config = create_cfgs() # merged_config = config.AnnotatorConfig(task_config, global_config) TM = TaskManager(global_config["max_process_number"], global_config["max_task_in_queue"]) for idx in range(10): _, task_config = create_cfgs() merged_config = config.AnnotatorConfig(task_config, global_config) btc = BatchTrainCmd(db_config, merged_config) ret = TM.exec_command(btc) if ret: print("add task ok!") else: print("can not add task queue full!")
def post(self): global task_manager json_data = request.get_json(force=True) if json_data is None: abort_message("can not parse post data as json!") start_timestamp = json_data.get("start_timestamp", None) model_type = json_data.get("task_type", None) uid = json_data.get("user_uuid", None) did = json_data.get("dataset_uuid", None) if start_timestamp is None or model_type is None or uid is None or did is None: abort_message( "can not find uid or did or task type or start label timestamp" ) db_config = GLOBAL_CONFIG[DB_CONFIG_KEY].copy() # TODO more task type will add later task_config = config.CLASSIFY_TASK_CONFIG.copy() batch_number = task_config.get("batch_num", DEFAULT_BATCH_NUM) task_config["user_uuid"] = uid task_config["dataset_uuid"] = did task_config["model_type"] = model_type task_config["condition"] = { "timestamp": { "$gt": datetime.datetime.fromtimestamp(float(start_timestamp)) } } task_config["sort_limit"] = ([("timestamp", pymongo.DESCENDING)], batch_number) task_config["model_version"] = time.time() # TODO pipline and embedding path will use user defined task_config["pipeline"] = [ "char_tokenizer", "sentence_embedding_extractor", "SVM_classifier" ] dir_name = os.path.realpath("../../") task_config[ "embedding_path"] = dir_name + "/tests/data/test_embedding/vec.txt" global_task_center_config = GLOBAL_CONFIG[TASK_CONFIG_KEY] merged_config = config.AnnotatorConfig(task_config, global_task_center_config) btc = BatchTrainCmd(db_config, merged_config) ret = task_manager.exec_command(btc) if ret: return jsonify({"status": "pendding", "task_id": btc.timestamp}) else: return jsonify({ "status": "error", "message": "queue full, can not add" })