def post(self):
json_data = request.get_json(force=True)
if json_data is None:
abort_message("can not parse post data as json!")
uid = json_data.get("user_uuid", None)
did = json_data.get("dataset_uuid", None)
task_type = json_data.get("task_type", None)
if uid is None or did is None or task_type is None:
abort_message(
"can not find user_uuid or dataset_uuid or task_type")
db_config = GLOBAL_CONFIG[DB_CONFIG_KEY].copy()
task_config = config.CLASSIFY_TASK_CONFIG.copy()
task_config["user_uuid"] = uid
task_config["dataset_uuid"] = did
task_config["model_type"] = task_type
global_task_center_config = GLOBAL_CONFIG[TASK_CONFIG_KEY]
merged_config = config.AnnotatorConfig(task_config,
global_task_center_config)
st_cmd = LatestStatusCmd(db_config, merged_config)
ret_data = st_cmd()
return jsonify({
"status": ret_data[0],
"end_time": ret_data[1],
"task_id": ret_data[2]
})
def post(self):
json_data = request.get_json(force=True)
if json_data is None:
abort_message("can not parse post data as json!")
model_type = json_data.get("task_type", None)
uid = json_data.get("user_uuid", None)
did = json_data.get("dataset_uuid", None)
data = json_data.get("data", None)
if model_type is None or uid is None or did is None or data is None:
abort_message("can not find uid or did or task type or data!")
db_config = GLOBAL_CONFIG[DB_CONFIG_KEY].copy()
# TODO more task type will add later
task_config = config.CLASSIFY_TASK_CONFIG.copy()
task_config["user_uuid"] = uid
task_config["dataset_uuid"] = did
task_config["model_type"] = model_type
task_config["data"] = data
global_task_center_config = GLOBAL_CONFIG[TASK_CONFIG_KEY]
merged_config = config.AnnotatorConfig(task_config,
global_task_center_config)
merged_config["model_path"] = merged_config.get_save_path_prefix()
bnpc = BatchNoDbPredictCmd(db_config, merged_config)
# format = {txt:XXX, result: {label : XXX, con: XXX}result
result = bnpc()
return jsonify(result)
def abs_test_status():
db_config = {"database_hostname":"localhost", "database_port" : 27017,
"database_type": "mongodb", "database_name": "chinese_annotator",
"user_name":"anno_admin", "password": "******"}
global_config, task_config = create_pred_cfgs()
merged_config = config.AnnotatorConfig(task_config, global_config)
st_cmd = StatusCmd(db_config, merged_config)
print(st_cmd())
def abc_test_batch_predict():
db_config = {"database_hostname":"localhost", "database_port" : 27017,
"database_type": "mongodb", "database_name": "chinese_annotator",
"user_name":"anno_admin", "password": "******"}
global_config, task_config = create_pred_cfgs()
merged_config = config.AnnotatorConfig(task_config, global_config)
merged_config["model_path"] = merged_config.get_save_path_prefix()
# TM = TaskManager(global_config["max_process_number"], global_config["max_task_in_queue"])
btc = BatchPredictCmd(db_config, merged_config)
preds = btc()
print(preds)
def abc_test_batch_train():
db_config = {"database_hostname":"localhost", "database_port" : 27017,
"database_type": "mongodb", "database_name": "chinese_annotator",
"user_name":"anno_admin", "password": "******"}
global_config, task_config = create_cfgs()
# merged_config = config.AnnotatorConfig(task_config, global_config)
TM = TaskManager(global_config["max_process_number"], global_config["max_task_in_queue"])
for idx in range(10):
_, task_config = create_cfgs()
merged_config = config.AnnotatorConfig(task_config, global_config)
btc = BatchTrainCmd(db_config, merged_config)
ret = TM.exec_command(btc)
if ret:
print("add task ok!")
else:
print("can not add task queue full!")
def post(self):
global task_manager
json_data = request.get_json(force=True)
if json_data is None:
abort_message("can not parse post data as json!")
start_timestamp = json_data.get("start_timestamp", None)
model_type = json_data.get("task_type", None)
uid = json_data.get("user_uuid", None)
did = json_data.get("dataset_uuid", None)
if start_timestamp is None or model_type is None or uid is None or did is None:
abort_message(
"can not find uid or did or task type or start label timestamp"
)
db_config = GLOBAL_CONFIG[DB_CONFIG_KEY].copy()
# TODO more task type will add later
task_config = config.CLASSIFY_TASK_CONFIG.copy()
batch_number = task_config.get("batch_num", DEFAULT_BATCH_NUM)
task_config["user_uuid"] = uid
task_config["dataset_uuid"] = did
task_config["model_type"] = model_type
task_config["condition"] = {
"timestamp": {
"$gt": datetime.datetime.fromtimestamp(float(start_timestamp))
}
}
task_config["sort_limit"] = ([("timestamp", pymongo.DESCENDING)],
batch_number)
task_config["model_version"] = time.time()
# TODO pipline and embedding path will use user defined
task_config["pipeline"] = [
"char_tokenizer", "sentence_embedding_extractor", "SVM_classifier"
]
dir_name = os.path.realpath("../../")
task_config[
"embedding_path"] = dir_name + "/tests/data/test_embedding/vec.txt"
global_task_center_config = GLOBAL_CONFIG[TASK_CONFIG_KEY]
merged_config = config.AnnotatorConfig(task_config,
global_task_center_config)
btc = BatchTrainCmd(db_config, merged_config)
ret = task_manager.exec_command(btc)
if ret:
return jsonify({"status": "pendding", "task_id": btc.timestamp})
else:
return jsonify({
"status": "error",
"message": "queue full, can not add"
})