def attempt(clf, X_train, y_train, X_test, y_test):
print clf
print 'weighted:'
_attempt(clf, X_train, y_train, X_test, y_test)
print
print 'weighted with undersampled test set:'
X_u_small, X_u_large, y_u = get_asym_task(X_test, y_test)
X_u_large = np.array(random.sample(X_u_large, len(X_u_small)))
X_u = np.vstack((X_u_small, X_u_large))
_attempt(clf, X_train, y_train, X_u, y_u, False)
print
print
def _run(repos, features):
"""Train and run a classifier using features from these repos.
Current classes are used.
:param repos: a list of Repos
:param features: a list of strings of feature names
"""
class_to_id, id_to_class = utils.create_bimap(classes.classes)
y = np.array([class_to_id[classes.classify(r)] for r in repos])
# all features except imports are numerical;
# imports become one-hot boolean ngrams
use_imports = False
if 'imported_stdlib_modules' in features:
use_imports = True
# mod_feature_dict = {_mod_feature_name(mods): False
# for mods in ngrams(sorted_stdlib_names)}
features = [f for f in features if f != 'imported_stdlib_modules']
dict_repos = []
for r in repos:
d = {}
if use_imports:
# d = mod_feature_dict.copy()
mods = [m for m in r.imported_stdlib_modules if m in
set(['hashlib', '__future__', 'functools', 'threading', 'warnings', 'base64',
'traceback', 'socket', 'urlparse', 'subprocess', 'tempfile', 'json',
'unittest', 'errno', 'StringIO', 're', 'glob', 'signal', 'inspect',
'operator'])]
for mods in ngrams(mods):
d[_mod_feature_name(mods)] = True
for fname in features:
d[fname] = getattr(r, fname)
dict_repos.append(d)
vec = DictVectorizer()
X = vec.fit_transform(dict_repos)
#X = X.todense()
feature_names = vec.get_feature_names()
dense_X = X.toarray()
# model search
X_train, X_test, y_train, y_test = sklearn.cross_validation.train_test_split(
dense_X, y, test_size=0.3
)
# rfc_grid = [
# {'max_features': [None, 'sqrt', 'log2'],
# 'criterion': ['entropy', 'gini'],
# 'n_estimators': [200, 500, 750],
# 'max_depth': [None],
# 'min_samples_split': [1, 2, 3, 5],
# },
# ]
# cv_rfc = GridSearchCV(RandomForestClassifier(),
# rfc_grid, cv=3, verbose=1, n_jobs=-1).fit(X_train, y_train)
# ada_grid = [
# {
# 'n_estimators': [200, 500, 750, 1000],
# 'algorithm': ['SAMME', 'SAMME.R']
# },
# ]
# cv_ada = GridSearchCV(AdaBoostClassifier(
# base_estimator=cv_rfc.best_estimator_.estimators_[0]),
# ada_grid, cv=3, verbose=1, n_jobs=-1).fit(X_train, y_train)
# print 'RFC 5-fold stratified'
# rfc = RandomForest()
# rfc.fit(X_train, y_train)
# pred = rfc.predict(X_test)
# print metrics.precision_recall_fscore_support(y_test, pred)
# #benchmark(RandomForest(), dense_X, y, feature_names)
# print 'RFC found by:'
# print cv_rfc.best_estimator_
# rfc = cv_rfc.best_estimator_
# rfc.fit(X_train, y_train)
# pred = rfc.predict(X_test)
# print metrics.precision_recall_fscore_support(y_test, pred)
# print 'ABC found by:'
# print cv_ada.best_estimator_
# rfc = cv_ada.best_estimator_
# rfc.fit(X_train, y_train)
# pred = rfc.predict(X_test)
# print metrics.precision_recall_fscore_support(y_test, pred)
#print 'Gradient boost'
#benchmark(GradientBoostingClassifier(n_estimators=300,
# max_depth=5,
# min_samples_split=1,
# max_features=None,
# ),
# dense_X, y, feature_names)
#size = .3
#print '5-fold strat %s' % size
#cv = sklearn.cross_validation.StratifiedShuffleSplit(
# y,
# n_iter=5,
# test_size=size
#)
#size = .5
#print '5-fold strat %s' % size
#benchmark(RandomForest(), dense_X, y, feature_names,
# cv=sklearn.cross_validation.StratifiedShuffleSplit(
# y,
# n_iter=5,
# test_size=size
# ))
def _attempt(clf, X_train, y_train, X_test, y_test, weighted=True):
weights = None
if weighted:
weights = balance_weights(y_train)
clf.fit(X_train, y_train, sample_weight=weights)
pred = clf.predict(X_test)
print metrics.classification_report(y_test, pred, target_names=['high', 'low'])
def attempt(clf, X_train, y_train, X_test, y_test):
print clf
print 'weighted:'
_attempt(clf, X_train, y_train, X_test, y_test)
print
print 'weighted with undersampled test set:'
X_u_small, X_u_large, y_u = get_asym_task(X_test, y_test)
X_u_large = np.array(random.sample(X_u_large, len(X_u_small)))
X_u = np.vstack((X_u_small, X_u_large))
_attempt(clf, X_train, y_train, X_u, y_u, False)
print
print
rfc = RandomForest()
attempt(rfc, X_train, y_train, X_test, y_test)
ada = AdaBoostClassifier(n_estimators=300)
attempt(ada, X_train, y_train, X_test, y_test)
#benchmark(RandomForest(), X_new, y_new, feature_names)
asym = AsymBaggingRFCs(13,
n_estimators=200,
max_depth=None,
min_samples_split=1,
max_features=None,
#random_state=0, # random seed is static for comparison
compute_importances=True,
n_jobs=-1, # run on all cores
)
attempt(asym, X_train, y_train, X_test, y_test)
print
print '============'
print 'with undersampled training data:'
rfc_under = RandomForest()
X_utr_small, X_utr_large, y_utr = get_asym_task(X_train, y_train)
X_utr_large = np.array(random.sample(X_utr_large, len(X_utr_small)))
X_utr = np.vstack((X_utr_small, X_utr_large))
attempt(rfc_under, X_utr, y_utr, X_test, y_test)
ada_under = AdaBoostClassifier(n_estimators=300)
attempt(ada_under, X_utr, y_utr, X_test, y_test)