def load_topic_tokens(file_path, encoding='latin-1', strip_html_tags=True, strip_html_entities=True, strip_square_bracket_tags=True, preprocess=create_preprocessor()): """Generator which provides a list of topics along with the preprocessed and tokenized fields """ result = [] topics = __regex_parse_topics_from_file(file_path) for topic in topics: processed = {} for field in ['title', 'narr', 'desc']: content = topic[field] words = split_words( content, strip_html_tags=strip_html_tags, strip_html_entities=strip_html_entities, strip_square_bracket_tags=strip_square_bracket_tags) terms = preprocess(words) processed[field] = set(terms) result.append( Topic(topic['id'], processed['title'], processed['narr'], processed['desc'])) return result
def run_eval(ranking_method, params={}): preprocessor = create_preprocessor( enable_case_folding=enable_case_folding, enable_remove_stop_words=enable_remove_stop_words, enable_stemmer=enable_stemmer, enable_lemmatizer=enable_lemmatizer, min_length=min_word_length) click.echo(f'Loading topics from {topics_file}') topics = load_topic_tokens( topics_file, preprocess=preprocessor, strip_html_tags=enable_strip_html_tags, strip_html_entities=enable_strip_html_entities, strip_square_bracket_tags=enable_strip_square_bracket_tags) click.echo('done') click.echo(f'Loading document stats from {stats_file}') document_stats = load_document_stats(stats_file) click.echo('done') click.echo(f'Loading search index from {index_file}') click.echo('This might take a while') start = time.time() number_of_documents, index_reader_generator = create_index_reader( index_file) index_reader = index_reader_generator() index = list(index_reader) click.echo(f'done in {time.time() - start} seconds') generate_qrel(number_of_documents, index, document_stats, topics, output_file, ranking_method, run_name, params)
def use_model(dataset, params, saved_model_filename): feature_fields = deepcopy(params['features']) feature_fields.append(params['index']) real_features, y = get_features_and_labels(dataset, feature_fields) preprocess_data_func = preprocess_data_function() try: myMod = compile(preprocess_data_func, '', 'exec') exec(myMod, globals()) real_features = preprocess_data(real_features) except Exception: print("System cannot run user code") real_X = real_features[params['features']] # create preprocessor for pipeline preprocessor = create_preprocessor(real_X) # load model pipeline = pickle.load(open(saved_model_filename, 'rb')) predictions = pipeline.predict(real_X) # print(real_features[params['index']]) # print(real_X[params['index']]) output = pd.DataFrame() output = pd.DataFrame({ params['index']: real_features[params['index']], params['label']: predictions }) return output
def cli(ctx, document_folder, index_file, stats_file, enable_case_folding, enable_stemmer, enable_lemmatizer, enable_remove_stop_words, min_word_length, enable_strip_html_tags, enable_strip_html_entities, enable_strip_square_bracket_tags): nltk.download('wordnet') preprocessor = create_preprocessor(enable_case_folding=enable_case_folding, enable_remove_stop_words=enable_remove_stop_words, enable_stemmer=enable_stemmer, enable_lemmatizer=enable_lemmatizer, min_length=min_word_length) glob_pattern = document_folder + '/**' document_files = [fname for fname in glob.glob(glob_pattern, recursive=True) if os.path.isfile(fname)] click.echo() click.echo('Processing {} file(s)'.format(len(document_files))) click.echo() ctx.obj['INDEX_FILE'] = index_file ctx.obj['STATS_FILE'] = stats_file ctx.obj['DOCUMENT_FILES'] = document_files ctx.obj['PREPROCESSOR'] = preprocessor ctx.obj['STRIP_HTML_TAGS'] = enable_strip_html_tags ctx.obj['STRIP_HTML_ENTITIES'] = enable_strip_html_entities ctx.obj['STRIP_SQUARE_BRACKET_TAGS'] = enable_strip_square_bracket_tags
def generate_tokens_for_files(filepaths, encoding='latin-1', use_regex_parser=True, strip_html_tags=True, strip_html_entities=True, strip_square_bracket_tags=True, preprocess=create_preprocessor()): """Generator which provides a list of (doc_id, term) pairs for documents contained in the given files """ num_documents_processed = 0 for filepath in tqdm(filepaths, total=len(filepaths)): documents = None if use_regex_parser: documents = __regex_parse_documents_from_file(filepath) else: documents = __xml_parse_documents_from_file(filepath) for document in documents: num_documents_processed += 1 (doc_id, content) = document words = split_words( content, strip_html_tags=strip_html_tags, strip_html_entities=strip_html_entities, strip_square_bracket_tags=strip_square_bracket_tags) terms = preprocess(words) for term in terms: yield (doc_id, term, num_documents_processed)
def run_eval(ranking_method, params={}): preprocess = create_preprocessor( enable_case_folding=enable_case_folding, enable_remove_stop_words=enable_remove_stop_words, enable_stemmer=enable_stemmer, enable_lemmatizer=enable_lemmatizer, min_length=min_word_length) words = split_words( query, strip_html_tags=enable_strip_html_tags, strip_html_entities=enable_strip_html_entities, strip_square_bracket_tags=enable_strip_square_bracket_tags) search_terms = preprocess(words) click.echo(f'Searching for "{query}" using "{ranking_method}"') click.echo(f'Words: "{words}"') click.echo(f'Terms: "{search_terms}"') click.echo(f'Loading document stats from {stats_file}') document_stats = load_document_stats(stats_file) click.echo('done') click.echo(f'Loading search index from {index_file}') click.echo('This might take a while') start = time.time() number_of_documents, index_reader_generator = create_index_reader( index_file) index_reader = index_reader_generator() index = list(index_reader) click.echo(f'done in {time.time() - start} seconds') document_scores = None if ranking_method == 'tfidf': document_scores = simple_tfidf_search(number_of_documents, index, search_terms) elif ranking_method == 'cosine_tfidf': document_scores = cosine_tfidf_search(number_of_documents, index, search_terms) elif ranking_method == 'bm25': document_scores = simple_bm25_search(number_of_documents, index, search_terms, document_stats, k1=params['k1'], b=params['b'], k3=params['k3']) elif ranking_method == 'bm25va': document_scores = simple_bm25va_search(number_of_documents, index, search_terms, document_stats, k1=params['k1'], k3=params['k3']) for document_score in document_scores[:50]: print(f'{document_score[1]}\t{document_score[0]}')
def add_preprocessor(): """Handle request for adding a new preprocessor. Returns: An string ID of the preprocessor. """ json_data = _get_json_from_request() error_log = validate_cfg(json_data, STATE, prepro=True) if error_log != {}: return json.dumps({'log': error_log}) prepro = create_preprocessor(json_data) idx = STATE.add_preprocessor(prepro) return json.dumps({'id': idx})
def train_model(dataset, fields, saved_model_filename, algorithm='decision_tree', task_type='classification'): data = pd.read_csv(dataset) # get features and labels features, y_train = get_features_and_labels(data, fields['features'], fields['label'], task_type=task_type) preprocess_data_func = preprocess_data_function() try: myMod = compile(preprocess_data_func, '', 'exec') exec(myMod, globals()) features = preprocess_data(features) except Exception: print("System cannot run user code") # features = preprocess_data(features) X_train = features[fields['features']] # create preprocessor for pipeline preprocessor = create_preprocessor(X_train) # train model if task_type == 'classification': if algorithm == 'decision_tree': model = DecisionTreeClassifier(random_state=241) elif algorithm == 'logistic_regression': model = LogisticRegression() elif algorithm == 'knn': model = KNeighborsClassifier() elif algorithm == 'svm': model = svm.SVC(random_state=241) elif algorithm == 'naive_bayes': model = GaussianNB() elif algorithm == 'sgd': model = SGDClassifier(random_state=241) elif algorithm == 'mlp': model = MLPClassifier(random_state=241) elif algorithm == 'rf': model = RandomForestClassifier(random_state=241) elif algorithm == 'gradient_boosting': model = GradientBoostingClassifier(random_state=241) if task_type == 'regression': if algorithm == 'decision_tree': model = DecisionTreeRegressor(random_state=241) elif algorithm == 'linear_regression': model = LinearRegression() elif algorithm == 'knn': model = KNeighborsRegressor() elif algorithm == 'svm': model = svm.SVR(random_state=241) elif algorithm == 'mlp': model = MLPRegressor(random_state=241) elif algorithm == 'rf': model = RandomForestRegressor(random_state=241) elif algorithm == 'gradient_boosting': model = GradientBoostingRegressor(random_state=241) scaler_name = 'robust' if scaler_name == 'robust': scaler = RobustScaler() elif scaler_name == 'min_max': scaler = MinMaxScaler() elif scaler_name == 'max_abs': scaler = MaxAbsScaler() elif scaler_name == 'standard': scaler = StandardScaler() pipeline = Pipeline(steps=[('preprocessor', preprocessor), ('scaler', scaler), ('model', model)]) pipeline.fit(X_train, y_train) # save the model to disk pickle.dump(pipeline, open(saved_model_filename, 'wb')) scoring = None if task_type == 'regression': scoring = 'neg_mean_absolute_error' score = cross_val_score(pipeline, X_train, y_train, cv=5, scoring=scoring) return score
from preprocessing import create_preprocessor, split_words from evaluation import generate_qrel, load_topic_tokens from indexing import create_index_reader, load_document_stats import gc import time index_filepath = 'spimi.index' stats_filepath = 'spimi.stats' topics_filepath = './data/TREC8all/topicsTREC8Adhoc.txt' preprocessor = create_preprocessor(enable_case_folding=True, enable_remove_stop_words=True, enable_stemmer=True, enable_lemmatizer=False, min_length=2) print('Loading topics from', topics_filepath) topics = load_topic_tokens(topics_filepath, preprocess=preprocessor) print('Searching', len(topics), 'topics') print('Loading document stats') document_stats = load_document_stats(stats_filepath) print('done') print('Loading search index') start = time.time() number_of_documents, index_reader_generator = create_index_reader(index_filepath) index_reader = index_reader_generator() index = list(index_reader) print('done in', time.time() - start, 'seconds')
def generate_tokens_for_files_distributed(filepaths, encoding='latin-1', use_regex_parser=True, strip_html_tags=True, strip_html_entities=True, strip_square_bracket_tags=True, preprocess=create_preprocessor()): """Generator which provides a list of (doc_id, term) pairs for documents contained in the given files """ num_documents_processed = 0 segment_path = "./segmented_files/" partitions = ["aa", "bc", "de", "fh", "ij", "km", "nq", "rs", "tu", "vz"] segments = [] for _ in partitions: segments.append([]) for filepath in tqdm(filepaths, total=len(filepaths)): documents = None if use_regex_parser: documents = __regex_parse_documents_from_file(filepath) else: documents = __xml_parse_documents_from_file(filepath) for document in documents: num_documents_processed += 1 (doc_id, content) = document words = split_words( content, strip_html_tags=strip_html_tags, strip_html_entities=strip_html_entities, strip_square_bracket_tags=strip_square_bracket_tags) terms = preprocess(words) for term in terms: c = term[0] for index, partition in enumerate(partitions): if index == segments.__len__() - 1: segments[index].append(term + " " + doc_id) break else: if c <= partition[1]: segments[index].append(term + " " + doc_id) break with open(segment_path + "meta_" + process_id().__str__(), "a") as file: file.write("{}\n".format(num_documents_processed)) for index, segment in enumerate(segments): #write tokenized documents to file file = open( segment_path + partitions[index] + "_" + process_id().__str__(), "a") for t in segment: file.write(t.__str__()) file.write("\n") file.flush() file.close()