def load_words(filepath: str): exist_words = get_exists_words() xhe_transformer = get_full_to_xhe_transformer() zrm_transformer = get_full_to_zrm_transformmer() lu_transformer = get_full_to_lu_transformmer() bingji_transformer = get_full_to_bingji_transformer() words = [] with open(filepath, 'r', encoding='utf8') as fin: for line in fin: line = line.strip() if len(line) == 0: continue cols = line.split(" ") if len(cols) > 5: print(f"wrong line {line}") continue if contain_alpha(cols[0]) or contain_symbols(cols[0]): print(f"contains num or symbols {line}") continue if cols[0] in exist_words: continue words.append( cols_to_tangshi_item(cols, xhe_transformer, zrm_transformer, bingji_transformer, lu_transformer)) exist_words.add(cols[0]) return words
_, words_path = sys.argv exist_words = set() exist_words = pipe(WordPhoneTable.select(), map(lambda e: e.word), set ) exist_words = exist_words | pipe(DelWordTable.select(), map(lambda e: e.word), set ) xhe_transformer = get_full_to_xhe_transformer(); zrm_transformer = get_full_to_zrm_transformmer(); lu_transformer = get_full_to_lu_transformmer(); with open(words_path, "r", encoding='utf8') as fin: to_add_words = pipe(fin, map(lambda e: e.strip().split(' ')), # filter(lambda e: len(e) in (1, 2)), filter(lambda e: len(e[0]) <= 5), filter(lambda e: not contain_alpha( e[0]) and not contain_symbols(e[0])), filter(lambda e: e[0] not in exist_words), map(lambda e: cols_to_word_phone_table(e, xhe_transformer, zrm_transformer)) ) with db.atomic(): WordPhoneTable.bulk_create(to_add_words, batch_size=100)