parser.add_argument('-o', '--output_dir', default="models/", help='Path to the output folder') args = parser.parse_args() if args.w2v: glove = Glove.load() w2v = glove.get_dict() print('w2v dict size:', len(w2v)) with open(os.path.join(args.output_dir, 'w2v_full.pkl'), 'wb') as f: pickle.dump(w2v, f, protocol=pickle.HIGHEST_PROTOCOL) if args.index: X_train, y_train, X_test, y_test = get_train_test_data(merge=True) tokenizer = Tokenizer(num_words=MAX_FEATURES) X_data = pd.concat((X_train, X_test), ignore_index=True) tokenizer.fit_on_texts(X_data) print("Word index dict size:", len(tokenizer.word_index)) outfile = os.path.join(args.output_dir, 'word_index_full.pkl') print("...wrote to", outfile) with open(outfile, 'wb') as f: pickle.dump(tokenizer.word_index, f, protocol=pickle.HIGHEST_PROTOCOL) if not args.w2v: W2V_DICT_PATH = './models/w2v.pkl' with open(W2V_DICT_PATH, 'rb') as f:
from database.utils import get_train_test_data from pipelines.feature_extractor import get_feature_extractor from pipelines.models import get_ensemble_model from feature_extraction.features import get_glove_w2v from evaluation.metrics import class_report from copy import deepcopy import numpy as np import pandas as pd import pprint import time if __name__ == '__main__': w2v = get_glove_w2v() train_test_data = get_train_test_data() feature_extractor = get_feature_extractor(w2v) ensemble = get_ensemble_model(w2v) models = [ ("lr", LogisticRegression(C=0.1, penalty='l2', solver='lbfgs', n_jobs=-1)), ("nb", BernoulliNB(alpha=5.0)), ("rf", RandomForestClassifier(n_estimators=300, max_depth=10, min_samples_split=5, n_jobs=-1)), ("xgb", XGBClassifier(n_estimators=300, max_depth=8, n_jobs=-1)), ("et", ExtraTreesClassifier(n_estimators=300,