def setup_module(): global x_tr, y_tr, x_dv, y_dv, counts_tr, x_dv_pruned, x_tr_pruned global labels global vocab global X_tr, X_tr_var, X_dv_var, Y_tr, Y_dv, Y_tr_var, Y_dv_var y_tr, x_tr = preproc.read_data('lyrics-train.csv', preprocessor=preproc.bag_of_words) labels = set(y_tr) counts_tr = preproc.aggregate_counts(x_tr) y_dv, x_dv = preproc.read_data('lyrics-dev.csv', preprocessor=preproc.bag_of_words) x_tr_pruned, vocab = preproc.prune_vocabulary(counts_tr, x_tr, 10) x_dv_pruned, _ = preproc.prune_vocabulary(counts_tr, x_dv, 10) ## remove this, so people can run earlier tests X_tr = preproc.make_numpy(x_tr_pruned, vocab) X_dv = preproc.make_numpy(x_dv_pruned, vocab) label_set = sorted(list(set(y_tr))) Y_tr = np.array([label_set.index(y_i) for y_i in y_tr]) Y_dv = np.array([label_set.index(y_i) for y_i in y_dv]) X_tr_var = Variable(torch.from_numpy(X_tr.astype(np.float32))) X_dv_var = Variable(torch.from_numpy(X_dv.astype(np.float32))) Y_tr_var = Variable(torch.from_numpy(Y_tr)) Y_dv_var = Variable(torch.from_numpy(Y_dv))
def test_d1_4_prune(): global x_dv, counts_tr x_tr_pruned, vocab = preproc.prune_vocabulary(counts_tr, x_tr, 3) x_dv_pruned, vocab2 = preproc.prune_vocabulary(counts_tr, x_dv, 3) eq_(len(vocab), len(vocab2)) eq_(len(vocab), 11824) eq_(len(x_dv[95].keys()) - len(x_dv_pruned[95].keys()), 8)
def setup_module(): #global y_tr, x_tr, corpus_counts, labels, vocab #corpus_counts = get_corpus_counts(x_tr) global x_tr, y_tr, x_dv, y_dv, counts_tr, x_dv_pruned, x_tr_pruned, x_bl_pruned global labels global vocab y_tr, x_tr = preproc.read_data('lyrics-train.csv', preprocessor=preproc.bag_of_words) labels = set(y_tr) counts_tr = preproc.aggregate_counts(x_tr) y_dv, x_dv = preproc.read_data('lyrics-dev.csv', preprocessor=preproc.bag_of_words) x_tr_pruned, vocab = preproc.prune_vocabulary(counts_tr, x_tr, 10) x_dv_pruned, _ = preproc.prune_vocabulary(counts_tr, x_dv, 10)
def test_d5_1_numpy(): global x_dv, counts_tr x_dv_pruned, vocab = preproc.prune_vocabulary(counts_tr, x_dv, 10) X_dv = preproc.make_numpy(x_dv_pruned, vocab) eq_(X_dv.sum(), 137687) eq_(X_dv.sum(axis=1)[4], 417) eq_(X_dv.sum(axis=1)[144], 175) eq_(X_dv.sum(axis=0)[10], 3) eq_(X_dv.sum(axis=0)[100], 0)