def test_d4_2a_perc_estimate(): global y_dv, x_tr_pruned, y_tr # run on a subset of data theta_perc,theta_perc_history = perceptron.estimate_perceptron(x_tr_pruned[:10],y_tr[:10],3) eq_(theta_perc[('2000s','its')],-1) eq_(theta_perc[('2000s','what')],1) eq_(theta_perc[('1980s','what')],4) eq_(theta_perc[('1980s','its')],-15) eq_(theta_perc_history[0][('1980s','what')],2)
def test_perc_d4_2(): global y_dv, x_tr, y_tr # run on a subset of data theta_perc,theta_perc_history = perceptron.estimate_perceptron(x_tr[:10],y_tr[:10],3) eq_(theta_perc[('worldnews','its')],1) eq_(theta_perc[('science','its')],0) eq_(theta_perc[('science','what')],4) eq_(theta_perc[('worldnews','always')],-1) eq_(theta_perc_history[0][('science','what')],2) y_hat_dv = evaluation.read_predictions('perc-dev.preds') # i get 64.6% accuracy assert_greater_equal(evaluation.acc(y_hat_dv,y_dv),.62)
'reddit-train.csv', #filename 'subreddit', #label field preprocessor=preproc.tokenize_and_downcase) #your preprocessor y_dv, x_dv = preproc.read_data( 'reddit-dev.csv', #filename 'subreddit', #label field preprocessor=preproc.tokenize_and_downcase) #your preprocessor y_te, x_te = preproc.read_data( 'reddit-test.csv', #filename 'subreddit', #label field preprocessor=preproc.tokenize_and_downcase) #your preprocessor corpus_counts = preproc.get_corpus_counts(x_tr) vocab = [word for word, count in corpus_counts.iteritems() if count > 10] print len(vocab) print len(x_tr[0]) x_tr = [{key: val for key, val in x_i.iteritems() if key in vocab} for x_i in x_tr] x_dv = [{key: val for key, val in x_i.iteritems() if key in vocab} for x_i in x_dv] x_te = [{key: val for key, val in x_i.iteritems() if key in vocab} for x_i in x_te] from gtnlplib import perceptron reload(perceptron) theta_perc, theta_perc_history = perceptron.estimate_perceptron(x_tr, y_tr, 20)