def check_train_error(): print '...load the expriment data set' data = cPickle.load(open('./data/experiment_dataset2')) docs, type2id, pop2id, word2id, embedding, rand_embedding = data print '... construct the train/valid/test set' test_docs = docs[:10000] datasets = make_data_cv(test_docs, 0, word2id, max_l=1000, filter_h=5) print '....Load model parameters' # load the trained parameters params = load_model('./data/pop_model.pkl') print '....start test the model' # construct the model rs = construct_model(params, datasets, filter_hs=[3, 4, 5], batch_size=200) rs["pop2id"] = pop2id with open('./rs.json', 'w') as r: json.dump(rs, r)
def check_train_error(): print '...load the expriment data set' data = cPickle.load(open('./data/experiment_dataset2')) docs, type2id, pop2id, word2id, embedding, rand_embedding = data print '... construct the train/valid/test set' test_docs = docs[:10000] datasets = make_data_cv(test_docs, 0, word2id, max_l=1000, filter_h=5) print '....Load model parameters' # load the trained parameters params= load_model('./data/pop_model.pkl') print '....start test the model' # construct the model rs = construct_model(params, datasets, filter_hs=[3,4,5], batch_size=200) rs["pop2id"] = pop2id with open('./rs.json', 'w') as r: json.dump(rs, r)
def main(): print '...load the expriment data set' data = cPickle.load(open('./data/experiment_dataset2')) docs, type2id, pop2id, word2id, embedding, rand_embedding = data print '... construct the train/valid/test set' test_docs = docs[:10000] datasets = make_data_cv(test_docs, 0, word2id, max_l=1000, filter_h=5) print '... construct ngrams' ns = [3, 4, 5] n_grams = find_ngrams(datasets[0]) print '....Load model parameters' # load the trained parameters params = load_model('./data/pop_model.pkl') print '....start test the model' # construct the model # dump the result with open("./data/top_ngrams.pkl", 'wb') as tn: for n, n_gram in enumerate(n_grams): n_gram_texts = get_top_features(n_gram, params, word2id, n) cPickle.dump(n_gram_texts, tn)
def main(): print '...load the expriment data set' data = cPickle.load(open('./data/experiment_dataset2')) docs, type2id, pop2id, word2id, embedding, rand_embedding = data print '... construct the train/valid/test set' test_docs = docs[:10000] datasets = make_data_cv(test_docs, 0, word2id, max_l=1000, filter_h=5) print '... construct ngrams' ns = [3, 4, 5] n_grams = find_ngrams(datasets[0]) print '....Load model parameters' # load the trained parameters params= load_model('./data/pop_model.pkl') print '....start test the model' # construct the model # dump the result with open("./data/top_ngrams.pkl", 'wb') as tn: for n, n_gram in enumerate(n_grams): n_gram_texts = get_top_features(n_gram, params, word2id, n) cPickle.dump(n_gram_texts, tn)