async def create_model_new_service(item: Item):
state = SUCCESS_CODE
record_id = -1
try:
name = item.name
domain = item.domain
categories = item.categories
conn = get_mysql_connect()
record_id = insert_model_record_new(conn, name, domain, STATE_TRAINING_NUMBER, categories, "")
model_dir = "./output/{}/{}_{}".format(domain, name, record_id)
if not os.path.exists(model_dir):
os.makedirs(model_dir, exist_ok=True)
label2answer = {}
with open(os.path.join(model_dir, "data.tsv"), mode="w") as f:
for category in categories:
texts, _ = get_category_queries(conn, category)
answer = get_category_answer(conn, category)
label2answer[category] = answer
for text in texts:
f.write("{}\t{}\n".format(text, category))
with open(os.path.join(model_dir, "label2answer.json"), mode="w") as f:
json.dump(label2answer, f, indent=2, ensure_ascii=False)
os.system("sh auto_train.sh {} {} {} {}".format(
record_id, name, domain, os.path.join(model_dir, "data.tsv")))
except Exception:
traceback.print_exc()
state = FAIL_CODE
return {"record_id": record_id, "state": state}
async def delete_query_service(item: DataItem):
state = SUCCESS_CODE
try:
query_id = str(item.query_id)
conn = get_mysql_connect()
delete_query(conn, query_id)
except Exception:
traceback.print_exc()
state = FAIL_CODE
return {"state": state}
async def create_query_service(item: DataItem):
state = SUCCESS_CODE
try:
category_id = item.category_id
text = item.text
conn = get_mysql_connect()
query_id = insert_query(conn, category_id, text)
except Exception:
traceback.print_exc()
state = FAIL_CODE
return {"query_id": query_id, "state": state}
async def delete_category_service(item: DataItem):
state = SUCCESS_CODE
try:
category_id = str(item.category_id)
conn = get_mysql_connect()
if delete_category(conn, category_id) == STATE_ERROR_NUMBER:
state = FAIL_CODE
except Exception:
traceback.print_exc()
state = FAIL_CODE
return {"state": state}
async def update_category_service(item: DataItem):
state = SUCCESS_CODE
try:
category_id = item.category_id
answer = item.answer
conn = get_mysql_connect()
state = update_category(conn, category_id, answer)
except Exception:
traceback.print_exc()
state = FAIL_CODE
return {"state": state}
async def create_category_service(item: DataItem):
state = SUCCESS_CODE
category_id = -1
try:
name = item.name
answer = item.answer
conn = get_mysql_connect()
category_id = insert_category(conn, name, answer)
except Exception:
traceback.print_exc()
state = FAIL_CODE
return {"category_id": category_id, "state": state}
async def delete_model_service(item: Item):
state = SUCCESS_CODE
model_state = STATE_READY_NUMBER
try:
record_id = item.record_id
conn = get_mysql_connect()
res = delete_model_record(conn, record_id)
if res != STATE_READY_NUMBER:
state = FAIL_CODE
model_state = res
except Exception:
traceback.print_exc()
state = FAIL_CODE
model_state = STATE_ERROR_NUMBER
return {"state": state, "model_state": model_state}
async def update_model_service(item: Item):
state = SUCCESS_CODE
try:
record_id = item.record_id
name = item.name
domain = item.domain
global state_map
old_record_id = state_map[domain].get("record_id", -1)
model_dir = "./output/{}/{}_{}".format(domain, name, record_id)
state_map[domain]["model"] = None
state_map[domain] = load_model(model_dir)
conn = get_mysql_connect()
update_model_record(conn, old_record_id, STATE_READY_NUMBER)
update_model_record(conn, record_id, STATE_USING_NUMBER)
except Exception:
traceback.print_exc()
state = FAIL_CODE
return {"state": state}
async def create_model_service(item: Item):
state = SUCCESS_CODE
record_id = -1
try:
name = item.name
domain = item.domain
data_path = item.data_path
category_num = item.category_num
conn = get_mysql_connect()
record_id = insert_model_record(conn, name, domain, STATE_TRAINING_NUMBER, data_path, category_num, "")
model_dir = "./output/{}/{}_{}".format(domain, name, record_id)
if not os.path.exists(model_dir):
os.makedirs(model_dir, exist_ok=True)
os.system("cp {} {}".format(data_path, os.path.join(model_dir, "data.tsv")))
os.system("sh auto_train.sh {} {} {} {}".format(record_id, name, domain, data_path))
except Exception:
traceback.print_exc()
state = FAIL_CODE
return {"record_id": record_id, "state": state}
doc_vecs, doc_labels = create_data_vec(data_list, model, tokenizer,
args["max_length"], device,
"response", True)
saved_data = {"vecs": doc_vecs, "labels": doc_labels}
with open(os.path.join(args["output_dir"], "total.pkl"),
mode="wb") as f:
pickle.dump(saved_data, f)
logger.info("finish generate corpus vecs to " +
os.path.join(args["output_dir"], "total.pkl"))
key = True
except RuntimeError as e:
traceback.print_exc()
device = torch.device("cpu")
try_count -= 1
return 0 if key else -1
if __name__ == '__main__':
setup_seed(20)
main_args = get_args()
res = main(args=main_args)
conn = get_mysql_connect()
if conn is not None:
if res == 0:
update_model_record(conn, main_args["record_id"],
STATE_READY_NUMBER)
else:
update_model_record(conn, main_args["record_id"],
STATE_ERROR_NUMBER)
conn.close()
