def main_build(args): # load language model ft_model_filename = args.path_model print(f"Loading language model from: {ft_model_filename}") model = ck.load_FT_model(ft_model_filename) print("Loaded embeddings!") # build new embedder es_embedder = ck.SIFEmbedder(model) print("Built embedder!") # get keywords keyword_filename = args.path_keywords print(f"Loading keywords from: {keyword_filename}") keywords = ck.load_csv_column( keyword_filename, args.keywords_column, delimiter = args.keywords_delimiter) print(f'Loaded {len(keywords)} keywords.') # run embedder es_embedder.fit(keywords, sample_size=args.sample) # store parameters embedder_params_filename = args.path_embedder_parameters print(f"Dumping embedder parameters to: {embedder_params_filename}") with open(embedder_params_filename, "w") as outfile: outfile.write(es_embedder.serialize())
def main_build(args): # load language model ft_model_filename = args.path_model print(f"Loading language model from: {ft_model_filename}") model = ck.load_FT_model(ft_model_filename) print("Loaded embeddings!") # build new embedder es_embedder = ck.SIFEmbedder(model) print("Built embedder!") # get keywords keyword_filename = args.path_keywords print(f"Loading keywords from: {keyword_filename}") keywords = ck.load_csv_column(keyword_filename, args.keywords_column, delimiter=args.keywords_delimiter) print(f'Loaded {len(keywords)} keywords.') # run embedder embeddings = es_embedder.fit_embed(keywords) # store parameters embedder_params_filename = args.path_embedder_parameters print(f"Dumping embedder parameters to: {embedder_params_filename}") with open(embedder_params_filename, "w") as outfile: outfile.write(es_embedder.serialize()) # if specified, store embeddings if args.path_embeddings is not None: embeddings_path = args.path_embeddings print(f"Dumping embeddings to: {embeddings_path}") np.save(open(embeddings_path, 'wb'), embeddings)
def main_categorise(args): # load language model ft_model_filename = args.path_model print(f"Loading language model from: {ft_model_filename}") model = ck.load_FT_model(ft_model_filename) print("Loaded embeddings!") # build embedder embedder_parameters_filename = args.path_embedder_parameters print(f"Loading embedder parameters from: {embedder_parameters_filename}") de_embedder_parameters_json = open(embedder_parameters_filename).read() de_embedder = ck.SIFEmbedder(model) de_embedder.load(de_embedder_parameters_json) print("Built embedder!") # get categories categories_filename = args.path_categories print(f"Loading categories from: {categories_filename}") categories = ck.load_csv_column(categories_filename, args.categories_column, delimiter=',') category_ids = ck.load_csv_column(categories_filename, args.categories_id_column, delimiter=',') print(f'Loaded {len(categories)} categories.') # build categorizer categorizer = ck.Categorizer(de_embedder) categorizer.fit(categories, category_ids=category_ids) print("Categorizer built!") # get keywords keyword_filename = args.path_keywords print(f"Loading keywords from: {keyword_filename}") keywords = ck.load_csv_column(keyword_filename, args.keywords_column, delimiter=args.keywords_delimiter) print(f'Loaded {len(keywords)} keywords.') # run categorizer n_categories = args.n_categories keyword_categories = categorizer.categorize(keywords, n_categories=n_categories) output_filename = args.path_output print(f"Writing categories to: {output_filename}") with open(output_filename, "w", encoding="utf8") as outfile: outwriter = csv.writer(outfile, delimiter=",", quotechar='"') # write header out_header = ["keyword"] for cat_i in range(1, n_categories + 1): out_header.extend( [f"category{cat_i}", f"category{cat_i}_distance"]) outwriter.writerow(out_header) # write results row by row for keyword, categories in zip(keywords, keyword_categories): row = [f"{keyword}"] for category, distance in categories: row.extend([f"{category}", f"{distance}"]) outwriter.writerow(row) print("DONE!")
def main_categorise(args): # load language model ft_model_filename = args.path_model print(f"Loading language model from: {ft_model_filename}") model = ck.load_FT_model(ft_model_filename) print("Loaded embeddings!") # build embedder embedder_parameters_filename = args.path_embedder_parameters print(f"Loading embedder parameters from: {embedder_parameters_filename}") de_embedder_parameters_json = open(embedder_parameters_filename).read() de_embedder = ck.SIFEmbedder(model) de_embedder.load(de_embedder_parameters_json) print("Built embedder!") # get categories categories_filename = args.path_categories print(f"Loading categories from: {categories_filename}") categories = ck.load_csv_column(categories_filename, args.categories_column, delimiter=',') category_ids = ck.load_csv_column(categories_filename, args.categories_id_column, delimiter=',') print(f'Loaded {len(categories)} categories.') # build categorizer categorizer = ck.Categorizer(de_embedder) categorizer.fit(categories, category_ids=category_ids) print("Categorizer built!") # get keywords keyword_filename = args.path_keywords print(f"Loading keywords from: {keyword_filename}") keywords = ck.load_csv_column( keyword_filename, args.keywords_column, delimiter = args.keywords_delimiter) print(f'Loaded {len(keywords)} keywords.') # assigning closest 'n_categories' to each keyword if args.mode == "categorise_keywords": n_categories = args.n_categories keyword_categories = categorizer.categorize(keywords, n_categories=n_categories) output_filename = args.path_output print(f"Writing categories to: {output_filename}") with open(output_filename, "w", encoding="utf8") as outfile: outwriter = csv.writer(outfile, delimiter=",", quotechar='"') # write header out_header = ["keyword"] for cat_i in range(1, n_categories + 1): out_header.extend([f"category{cat_i}_id", f"category{cat_i}", f"category{cat_i}_distance"]) outwriter.writerow(out_header) # write results row by row for keyword, categories in zip(keywords, keyword_categories): row = [f"{keyword}"] for category, id, distance in categories: row.extend([f"{id}", f"{category}", f"{distance}"]) outwriter.writerow(row) # assigning closest 'n_keywords' to each category elif args.mode == "relevance_to_category": n_keywords = args.n_keywords keyword_categories = categorizer.closest_keywords(keywords, n_keywords=n_keywords) output_filename = args.path_output print(f"Writing categories to: {output_filename}") with open(output_filename, "w", encoding="utf8") as outfile: outwriter = csv.writer(outfile, delimiter=",", quotechar='"') # write header out_header = ["keyword", "categories"] outwriter.writerow(out_header) # write results row by row for keyword, categories in tqdm(zip(keywords, keyword_categories), desc='Writting to file'): row = [f"{keyword}"] if len(categories) == 0: row += ["none"] else: #row += ["#".join([f"{category}({id})" for category, id, distance in categories])] for category, id, distance in categories: tmp_row = row + [f"{category}({id})"] outwriter.writerow(tmp_row) print("DONE!")
) argparser.add_argument( '--categories_id_column', '-cic', type=str, default='CategoryID', help= 'Name of column containing category ids in the categories csv file. (default: \'CategoryID\')' ) argparser.add_argument("-p", "--port", type=int, default=5000) args = argparser.parse_args() # load language model ft_model_filename = args.path_model print(f"Loading language model from: {ft_model_filename}") model = ck.load_FT_model(ft_model_filename) print("Loaded embeddings!") # build embedder embedder_parameters_filename = args.path_embedder_parameters print(f"Loading embedder parameters from: {embedder_parameters_filename}") de_embedder_parameters_json = open(embedder_parameters_filename).read() de_embedder = ck.SIFEmbedder(model) de_embedder.load(de_embedder_parameters_json) print("Built embedder!") # get categories categories_filename = args.path_categories print(f"Loading categories from: {categories_filename}") categories = ck.load_csv_column(categories_filename, args.categories_column,
import cluster_keywords as ck import numpy as np import json import csv import re import pdb print("import ok!") model = ck.load_FT_model("/home/ew-shopp/fasttext/models/cc.de.300.bin") print("load_FT_embeddings ok!") keywords = ["one", "one two", "one two three"] word_frequencies = ck.count_word_frequencies(keywords) print(word_frequencies) embeddings, principal_components = ck.sif_embedding(keywords, model, word_frequencies, n_principal_components=1, alpha=1e-3, principal_components=None, return_components=True) print("embeddings[5,:]:\n", embeddings[:, :5]) print("principal_components[5,:]\n", principal_components[:, :5]) embedder = ck.SIFEmbedder(model)