def attempt(clf, X_train, y_train, X_test, y_test): print clf print 'weighted:' _attempt(clf, X_train, y_train, X_test, y_test) print print 'weighted with undersampled test set:' X_u_small, X_u_large, y_u = get_asym_task(X_test, y_test) X_u_large = np.array(random.sample(X_u_large, len(X_u_small))) X_u = np.vstack((X_u_small, X_u_large)) _attempt(clf, X_train, y_train, X_u, y_u, False) print print
def _run(repos, features): """Train and run a classifier using features from these repos. Current classes are used. :param repos: a list of Repos :param features: a list of strings of feature names """ class_to_id, id_to_class = utils.create_bimap(classes.classes) y = np.array([class_to_id[classes.classify(r)] for r in repos]) # all features except imports are numerical; # imports become one-hot boolean ngrams use_imports = False if 'imported_stdlib_modules' in features: use_imports = True # mod_feature_dict = {_mod_feature_name(mods): False # for mods in ngrams(sorted_stdlib_names)} features = [f for f in features if f != 'imported_stdlib_modules'] dict_repos = [] for r in repos: d = {} if use_imports: # d = mod_feature_dict.copy() mods = [m for m in r.imported_stdlib_modules if m in set(['hashlib', '__future__', 'functools', 'threading', 'warnings', 'base64', 'traceback', 'socket', 'urlparse', 'subprocess', 'tempfile', 'json', 'unittest', 'errno', 'StringIO', 're', 'glob', 'signal', 'inspect', 'operator'])] for mods in ngrams(mods): d[_mod_feature_name(mods)] = True for fname in features: d[fname] = getattr(r, fname) dict_repos.append(d) vec = DictVectorizer() X = vec.fit_transform(dict_repos) #X = X.todense() feature_names = vec.get_feature_names() dense_X = X.toarray() # model search X_train, X_test, y_train, y_test = sklearn.cross_validation.train_test_split( dense_X, y, test_size=0.3 ) # rfc_grid = [ # {'max_features': [None, 'sqrt', 'log2'], # 'criterion': ['entropy', 'gini'], # 'n_estimators': [200, 500, 750], # 'max_depth': [None], # 'min_samples_split': [1, 2, 3, 5], # }, # ] # cv_rfc = GridSearchCV(RandomForestClassifier(), # rfc_grid, cv=3, verbose=1, n_jobs=-1).fit(X_train, y_train) # ada_grid = [ # { # 'n_estimators': [200, 500, 750, 1000], # 'algorithm': ['SAMME', 'SAMME.R'] # }, # ] # cv_ada = GridSearchCV(AdaBoostClassifier( # base_estimator=cv_rfc.best_estimator_.estimators_[0]), # ada_grid, cv=3, verbose=1, n_jobs=-1).fit(X_train, y_train) # print 'RFC 5-fold stratified' # rfc = RandomForest() # rfc.fit(X_train, y_train) # pred = rfc.predict(X_test) # print metrics.precision_recall_fscore_support(y_test, pred) # #benchmark(RandomForest(), dense_X, y, feature_names) # print 'RFC found by:' # print cv_rfc.best_estimator_ # rfc = cv_rfc.best_estimator_ # rfc.fit(X_train, y_train) # pred = rfc.predict(X_test) # print metrics.precision_recall_fscore_support(y_test, pred) # print 'ABC found by:' # print cv_ada.best_estimator_ # rfc = cv_ada.best_estimator_ # rfc.fit(X_train, y_train) # pred = rfc.predict(X_test) # print metrics.precision_recall_fscore_support(y_test, pred) #print 'Gradient boost' #benchmark(GradientBoostingClassifier(n_estimators=300, # max_depth=5, # min_samples_split=1, # max_features=None, # ), # dense_X, y, feature_names) #size = .3 #print '5-fold strat %s' % size #cv = sklearn.cross_validation.StratifiedShuffleSplit( # y, # n_iter=5, # test_size=size #) #size = .5 #print '5-fold strat %s' % size #benchmark(RandomForest(), dense_X, y, feature_names, # cv=sklearn.cross_validation.StratifiedShuffleSplit( # y, # n_iter=5, # test_size=size # )) def _attempt(clf, X_train, y_train, X_test, y_test, weighted=True): weights = None if weighted: weights = balance_weights(y_train) clf.fit(X_train, y_train, sample_weight=weights) pred = clf.predict(X_test) print metrics.classification_report(y_test, pred, target_names=['high', 'low']) def attempt(clf, X_train, y_train, X_test, y_test): print clf print 'weighted:' _attempt(clf, X_train, y_train, X_test, y_test) print print 'weighted with undersampled test set:' X_u_small, X_u_large, y_u = get_asym_task(X_test, y_test) X_u_large = np.array(random.sample(X_u_large, len(X_u_small))) X_u = np.vstack((X_u_small, X_u_large)) _attempt(clf, X_train, y_train, X_u, y_u, False) print print rfc = RandomForest() attempt(rfc, X_train, y_train, X_test, y_test) ada = AdaBoostClassifier(n_estimators=300) attempt(ada, X_train, y_train, X_test, y_test) #benchmark(RandomForest(), X_new, y_new, feature_names) asym = AsymBaggingRFCs(13, n_estimators=200, max_depth=None, min_samples_split=1, max_features=None, #random_state=0, # random seed is static for comparison compute_importances=True, n_jobs=-1, # run on all cores ) attempt(asym, X_train, y_train, X_test, y_test) print print '============' print 'with undersampled training data:' rfc_under = RandomForest() X_utr_small, X_utr_large, y_utr = get_asym_task(X_train, y_train) X_utr_large = np.array(random.sample(X_utr_large, len(X_utr_small))) X_utr = np.vstack((X_utr_small, X_utr_large)) attempt(rfc_under, X_utr, y_utr, X_test, y_test) ada_under = AdaBoostClassifier(n_estimators=300) attempt(ada_under, X_utr, y_utr, X_test, y_test)