async def create_model_new_service(item: Item): state = SUCCESS_CODE record_id = -1 try: name = item.name domain = item.domain categories = item.categories conn = get_mysql_connect() record_id = insert_model_record_new(conn, name, domain, STATE_TRAINING_NUMBER, categories, "") model_dir = "./output/{}/{}_{}".format(domain, name, record_id) if not os.path.exists(model_dir): os.makedirs(model_dir, exist_ok=True) label2answer = {} with open(os.path.join(model_dir, "data.tsv"), mode="w") as f: for category in categories: texts, _ = get_category_queries(conn, category) answer = get_category_answer(conn, category) label2answer[category] = answer for text in texts: f.write("{}\t{}\n".format(text, category)) with open(os.path.join(model_dir, "label2answer.json"), mode="w") as f: json.dump(label2answer, f, indent=2, ensure_ascii=False) os.system("sh auto_train.sh {} {} {} {}".format( record_id, name, domain, os.path.join(model_dir, "data.tsv"))) except Exception: traceback.print_exc() state = FAIL_CODE return {"record_id": record_id, "state": state}
async def delete_query_service(item: DataItem): state = SUCCESS_CODE try: query_id = str(item.query_id) conn = get_mysql_connect() delete_query(conn, query_id) except Exception: traceback.print_exc() state = FAIL_CODE return {"state": state}
async def create_query_service(item: DataItem): state = SUCCESS_CODE try: category_id = item.category_id text = item.text conn = get_mysql_connect() query_id = insert_query(conn, category_id, text) except Exception: traceback.print_exc() state = FAIL_CODE return {"query_id": query_id, "state": state}
async def delete_category_service(item: DataItem): state = SUCCESS_CODE try: category_id = str(item.category_id) conn = get_mysql_connect() if delete_category(conn, category_id) == STATE_ERROR_NUMBER: state = FAIL_CODE except Exception: traceback.print_exc() state = FAIL_CODE return {"state": state}
async def update_category_service(item: DataItem): state = SUCCESS_CODE try: category_id = item.category_id answer = item.answer conn = get_mysql_connect() state = update_category(conn, category_id, answer) except Exception: traceback.print_exc() state = FAIL_CODE return {"state": state}
async def create_category_service(item: DataItem): state = SUCCESS_CODE category_id = -1 try: name = item.name answer = item.answer conn = get_mysql_connect() category_id = insert_category(conn, name, answer) except Exception: traceback.print_exc() state = FAIL_CODE return {"category_id": category_id, "state": state}
async def delete_model_service(item: Item): state = SUCCESS_CODE model_state = STATE_READY_NUMBER try: record_id = item.record_id conn = get_mysql_connect() res = delete_model_record(conn, record_id) if res != STATE_READY_NUMBER: state = FAIL_CODE model_state = res except Exception: traceback.print_exc() state = FAIL_CODE model_state = STATE_ERROR_NUMBER return {"state": state, "model_state": model_state}
async def update_model_service(item: Item): state = SUCCESS_CODE try: record_id = item.record_id name = item.name domain = item.domain global state_map old_record_id = state_map[domain].get("record_id", -1) model_dir = "./output/{}/{}_{}".format(domain, name, record_id) state_map[domain]["model"] = None state_map[domain] = load_model(model_dir) conn = get_mysql_connect() update_model_record(conn, old_record_id, STATE_READY_NUMBER) update_model_record(conn, record_id, STATE_USING_NUMBER) except Exception: traceback.print_exc() state = FAIL_CODE return {"state": state}
async def create_model_service(item: Item): state = SUCCESS_CODE record_id = -1 try: name = item.name domain = item.domain data_path = item.data_path category_num = item.category_num conn = get_mysql_connect() record_id = insert_model_record(conn, name, domain, STATE_TRAINING_NUMBER, data_path, category_num, "") model_dir = "./output/{}/{}_{}".format(domain, name, record_id) if not os.path.exists(model_dir): os.makedirs(model_dir, exist_ok=True) os.system("cp {} {}".format(data_path, os.path.join(model_dir, "data.tsv"))) os.system("sh auto_train.sh {} {} {} {}".format(record_id, name, domain, data_path)) except Exception: traceback.print_exc() state = FAIL_CODE return {"record_id": record_id, "state": state}
doc_vecs, doc_labels = create_data_vec(data_list, model, tokenizer, args["max_length"], device, "response", True) saved_data = {"vecs": doc_vecs, "labels": doc_labels} with open(os.path.join(args["output_dir"], "total.pkl"), mode="wb") as f: pickle.dump(saved_data, f) logger.info("finish generate corpus vecs to " + os.path.join(args["output_dir"], "total.pkl")) key = True except RuntimeError as e: traceback.print_exc() device = torch.device("cpu") try_count -= 1 return 0 if key else -1 if __name__ == '__main__': setup_seed(20) main_args = get_args() res = main(args=main_args) conn = get_mysql_connect() if conn is not None: if res == 0: update_model_record(conn, main_args["record_id"], STATE_READY_NUMBER) else: update_model_record(conn, main_args["record_id"], STATE_ERROR_NUMBER) conn.close()