def measure_prior(datasize=1000, testsize=500): data = cu.get_sample_data_frame(datasize) test = cu.get_test_data_frame(testsize) priors = cu.load_priors('train.csv') num_samples = len(test) probs = np.kron(np.ones((num_samples,1)), priors) y_true = compute_y_true(test) score = multiclass_log_loss(y_true, probs) return score
def measure_prior(datasize=1000, testsize=500): data = cu.get_sample_data_frame(datasize) test = cu.get_test_data_frame(testsize) priors = cu.load_priors("train.csv") num_samples = len(test) probs = np.kron(np.ones((num_samples, 1)), priors) y_true = compute_y_true(test) score = multiclass_log_loss(y_true, probs) return score
def measure_bayes(datasize=1000, testsize=500): data = cu.get_sample_data_frame(datasize) test = cu.get_test_data_frame(testsize) nbfd = features.naive_features(data) nbft = features.naive_features(test) nb = nltk.NaiveBayesClassifier.train(nbfd) probs = [] for i in range(len(nbft)): p = nb.prob_classify(nbft[i][0]) probs.append([p.prob(s) for s in p.samples()]) probs = np.array(probs) new_priors = cu.load_priors("train.csv") old_priors = cu.compute_priors(data.OpenStatus) probs = cu.cap_and_update_priors(old_priors, probs, new_priors, 0.001) y_true = compute_y_true(test) score = multiclass_log_loss(y_true, probs) return score, nb, y_true, probs
def measure_bayes(datasize=1000, testsize=500): data = cu.get_sample_data_frame(datasize) test = cu.get_test_data_frame(testsize) nbfd = features.naive_features(data) nbft = features.naive_features(test) nb = nltk.NaiveBayesClassifier.train(nbfd) probs = [] for i in range(len(nbft)): p = nb.prob_classify(nbft[i][0]) probs.append([p.prob(s) for s in p.samples()]) probs = np.array(probs) new_priors = cu.load_priors('train.csv') old_priors = cu.compute_priors(data.OpenStatus) probs = cu.cap_and_update_priors(old_priors, probs, new_priors, 0.001) y_true = compute_y_true(test) score = multiclass_log_loss(y_true, probs) return score, nb, y_true, probs
def measure_model(datasize=1000, testsize=500): data = cu.get_sample_data_frame(datasize) test = cu.get_test_data_frame(testsize) # data = full.ix[len(full)/4:].reset_index() # last n/4 * 3 records # test = full.ix[:(len(full)/4)-1].reset_index() # first n/4 records # data = cu.get_dataframe('train-sample.csv') # test = cu.get_dataframe('public_leaderboard.csv') fea = features.extract_features(data) test_features = features.extract_features(test) rf = RandomForestClassifier(n_estimators=50, verbose=2, compute_importances=True, n_jobs=5) rf.fit(fea, data["OpenStatus"]) probs = rf.predict_proba(test_features) new_priors = cu.load_priors("train.csv") old_priors = cu.compute_priors(data.OpenStatus) probs = cu.cap_and_update_priors(old_priors, probs, new_priors, 0.001) y_true = compute_y_true(test) score = multiclass_log_loss(y_true, probs) return score, rf, fea
def measure_model(datasize=1000, testsize=500): data = cu.get_sample_data_frame(datasize) test = cu.get_test_data_frame(testsize) #data = full.ix[len(full)/4:].reset_index() # last n/4 * 3 records #test = full.ix[:(len(full)/4)-1].reset_index() # first n/4 records #data = cu.get_dataframe('train-sample.csv') #test = cu.get_dataframe('public_leaderboard.csv') fea = features.extract_features(data) test_features = features.extract_features(test) rf = RandomForestClassifier(n_estimators=50, verbose=2, compute_importances=True, n_jobs=5) rf.fit(fea, data["OpenStatus"]) probs = rf.predict_proba(test_features) new_priors = cu.load_priors('train.csv') old_priors = cu.compute_priors(data.OpenStatus) probs = cu.cap_and_update_priors(old_priors, probs, new_priors, 0.001) y_true = compute_y_true(test) score = multiclass_log_loss(y_true, probs) return score, rf, fea
def measure_svm(datasize=1000, testsize=500): data = cu.get_sample_data_frame(datasize) test = cu.get_test_data_frame(testsize) vocab = [w.strip() for w in file("vocab4.txt")][0:1000] vidx = get_vocab_index_lookup(vocab) print "extracting data features" xdata = extract_svm_features(vidx, data) print "extracting test features" xtest = extract_svm_features(vidx, test) labels = sorted(cu.labels) ydata = data.OpenStatus.apply(labels.index).tolist() model = svm.sparse.SVC(probability=True) print "fitting model" model.fit(xdata, ydata) print "rest" probs = model.predict_proba(xtest) new_priors = cu.load_priors("train.csv") old_priors = cu.compute_priors(data.OpenStatus) probs = cu.cap_and_update_priors(old_priors, probs, new_priors, 0.001) y_true = compute_y_true(test) score = multiclass_log_loss(y_true, probs) return score, model
def measure_svm(datasize=1000, testsize=500): data = cu.get_sample_data_frame(datasize) test = cu.get_test_data_frame(testsize) vocab = [w.strip() for w in file('vocab4.txt')][0:1000] vidx = get_vocab_index_lookup(vocab) print 'extracting data features' xdata = extract_svm_features(vidx, data) print 'extracting test features' xtest = extract_svm_features(vidx, test) labels = sorted(cu.labels) ydata = data.OpenStatus.apply(labels.index).tolist() model = svm.sparse.SVC(probability=True) print 'fitting model' model.fit(xdata, ydata) print 'rest' probs = model.predict_proba(xtest) new_priors = cu.load_priors('train.csv') old_priors = cu.compute_priors(data.OpenStatus) probs = cu.cap_and_update_priors(old_priors, probs, new_priors, 0.001) y_true = compute_y_true(test) score = multiclass_log_loss(y_true, probs) return score, model
def measure_lda(datasize=1000, testsize=500): # The number of documents to analyze each iteration batchsize = 100 # The total number of questions on Stack Overflow D = 3.3e6 # The number of topics K = 100 # How many documents to look at documentstoanalyze = datasize / batchsize # Our vocabulary vocab = [w.strip() for w in file("./vocab2.txt")] W = len(vocab) # the data data = cu.get_sample_data_frame(datasize) test = cu.get_test_data_frame(testsize) # Initialize the algorithm with alpha=1/K, eta=1/K, tau_0=1024, kappa=0.7 lda = OnlineLDA(vocab, K, D, 1.0 / K, 1.0 / K, 1024.0, 0.7) make_topic_columns(lda, data, K, D, batchsize) make_topic_columns(lda, test, K, D, batchsize) # data = full.ix[len(full)/4:].reset_index() # last n/4 * 3 records # test = full.ix[:(len(full)/4)-1].reset_index() # first n/4 records # data = cu.get_dataframe('train-sample.csv') # test = cu.get_dataframe('public_leaderboard.csv') fea = features.extract_features(data) test_features = features.extract_features(test) rf = RandomForestClassifier(n_estimators=50, verbose=2, compute_importances=True, n_jobs=5) rf.fit(fea, data["OpenStatus"]) probs = rf.predict_proba(test_features) new_priors = cu.load_priors("train.csv") old_priors = cu.compute_priors(data.OpenStatus) probs = cu.cap_and_update_priors(old_priors, probs, new_priors, 0.001) y_true = compute_y_true(test) score = multiclass_log_loss(y_true, probs) return score, rf, fea, data, datagamma
def measure_lda(datasize=1000, testsize=500): # The number of documents to analyze each iteration batchsize = 100 # The total number of questions on Stack Overflow D = 3.3e6 # The number of topics K = 100 # How many documents to look at documentstoanalyze = datasize / batchsize # Our vocabulary vocab = [w.strip() for w in file('./vocab2.txt')] W = len(vocab) # the data data = cu.get_sample_data_frame(datasize) test = cu.get_test_data_frame(testsize) # Initialize the algorithm with alpha=1/K, eta=1/K, tau_0=1024, kappa=0.7 lda = OnlineLDA(vocab, K, D, 1./K, 1./K, 1024., 0.7) make_topic_columns(lda, data, K, D, batchsize) make_topic_columns(lda, test, K, D, batchsize) #data = full.ix[len(full)/4:].reset_index() # last n/4 * 3 records #test = full.ix[:(len(full)/4)-1].reset_index() # first n/4 records #data = cu.get_dataframe('train-sample.csv') #test = cu.get_dataframe('public_leaderboard.csv') fea = features.extract_features(data) test_features = features.extract_features(test) rf = RandomForestClassifier(n_estimators=50, verbose=2, compute_importances=True, n_jobs=5) rf.fit(fea, data["OpenStatus"]) probs = rf.predict_proba(test_features) new_priors = cu.load_priors('train.csv') old_priors = cu.compute_priors(data.OpenStatus) probs = cu.cap_and_update_priors(old_priors, probs, new_priors, 0.001) y_true = compute_y_true(test) score = multiclass_log_loss(y_true, probs) return score, rf, fea, data, datagamma