for clf, name in ((RidgeClassifier(tol=1e-1), "Ridge Classifier"),
(KNeighborsClassifier(n_neighbors=10), "kNN")):
print 80 * '='
print name
results = benchmark(clf)
for penalty in ["l2", "l1"]:
print 80 * '='
print "%s penalty" % penalty.upper()
# Train Liblinear model
liblinear_results = benchmark(
LinearSVC(loss='l2', penalty=penalty, C=1000, dual=False, tol=1e-3))
# Train SGD model
sgd_results = benchmark(
SGDClassifier(alpha=.0001, n_iter=50, penalty=penalty))
# Train SGD with Elastic Net penalty
print 80 * '='
print "Elastic-Net penalty"
sgd_results = benchmark(
SGDClassifier(alpha=.0001, n_iter=50, penalty="elasticnet"))
# Train sparse Naive Bayes classifiers
print 80 * '='
print "Naive Bayes"
mnnb_results = benchmark(MultinomialNB(alpha=.01))
bnb_result = benchmark(BernoulliNB(alpha=.01))
class L1LinearSVC(LinearSVC):
# X: feature matrix; y: result array; z_k: prediction result array for k's model # # Setup 10 fold cross validation fold_num = 10 kf = KFold(n_samples, k=fold_num, indices=True) # set number of neighbors for kNN n_neighb = 13 # Brute-force implementation clf_bNB = BernoulliNB(alpha=.01) clf_mNB = MultinomialNB(alpha=.01) clf_kNN = KNeighborsClassifier(n_neighbors=n_neighb) clf_ridge = RidgeClassifier(tol=1e-1) clf_SGD = SGDClassifier(alpha=.0001, n_iter=50, penalty="l2") clf_lSVC = LinearSVC(loss='l2', penalty='l2', C=1000, dual=False, tol=1e-3) clf_SVC = SVC(C=1024, kernel='rbf', degree=3, gamma=0.001, probability=True) ############################################################################### # Stacking # # initialize empty y and z print 'X_den shape: ', X_den.shape print 'y shape: ', y.shape n_categories = len(set(y)) z = np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=float) # z = np.zeros( (n_samples, n_categories) , dtype=float)
print "Loading 20 newsgroups dataset for categories:"
print categories
data = fetch_20newsgroups(subset='train', categories=categories)
print "%d documents" % len(data.filenames)
print "%d categories" % len(data.target_names)
print
################################################################################
# define a pipeline combining a text feature extractor with a simple
# classifier
pipeline = Pipeline([
('vect', CountVectorizer()),
('tfidf', TfidfTransformer()),
('clf', SGDClassifier()),
])
parameters = {
# uncommenting more parameters will give better exploring power but will
# increase processing time in a combinatorial way
'vect__max_df': (0.5, 0.75, 1.0),
# 'vect__max_features': (None, 5000, 10000, 50000),
'vect__analyzer__max_n': (1, 2), # words or bigrams
# 'tfidf__use_idf': (True, False),
# 'tfidf__norm': ('l1', 'l2'),
'clf__alpha': (0.00001, 0.000001),
'clf__penalty': ('l2', 'elasticnet'),
# 'clf__n_iter': (10, 50, 80),
}
for clf, name in ((RidgeClassifier(tol=1e-1), "Ridge Classifier"),
(KNeighborsClassifier(n_neighbors=10), "kNN")):
print 80 * '='
print name
results = benchmark(clf)
for penalty in ["l2", "l1"]:
print 80 * '='
print "%s penalty" % penalty.upper()
# Train Liblinear model
liblinear_results = benchmark(LinearSVC(loss='l2', penalty=penalty, C=1000,
dual=False, tol=1e-3))
# Train SGD model
sgd_results = benchmark(SGDClassifier(alpha=.0001, n_iter=50,
penalty=penalty))
# Train SGD with Elastic Net penalty
print 80 * '='
print "Elastic-Net penalty"
sgd_results = benchmark(SGDClassifier(alpha=.0001, n_iter=50,
penalty="elasticnet"))
# Train sparse Naive Bayes classifiers
print 80 * '='
print "Naive Bayes"
mnnb_results = benchmark(MultinomialNB(alpha=.01))
bnb_result = benchmark(BernoulliNB(alpha=.01))
class L1LinearSVC(LinearSVC):
# clfs.append(KNeighborsClassifier(n_neighbors=n_neighb)) # clfs.append(RidgeClassifier(tol=1e-1)) # # clfs.append(SGDClassifier(alpha=.0001, n_iter=50, penalty="l1")) # clfs.append(SGDClassifier(alpha=.0001, n_iter=50, penalty="l2")) # # clfs.append(SGDClassifier(alpha=.0001, n_iter=50, penalty="elasticnet")) # # clfs.append(LinearSVC(loss='l2', penalty='l1', C=1000, dual=False, tol=1e-3)) # clfs.append(LinearSVC(loss='l2', penalty='l2', C=1000, dual=False, tol=1e-3)) # clfs.append(SVC(C=1000)) # Brute-force implementation clf_bNB = BernoulliNB(alpha=.01) clf_mNB = MultinomialNB(alpha=.01) clf_kNN = KNeighborsClassifier(n_neighbors=n_neighb) clf_ridge = RidgeClassifier(tol=1e-1) # clfs.append(SGDClassifier(alpha=.0001, n_iter=50, penalty="l1")) clf_SGD = SGDClassifier(alpha=.0001, n_iter=50, penalty="l2") # clfs.append(SGDClassifier(alpha=.0001, n_iter=50, penalty="elasticnet")) # clfs.append(LinearSVC(loss='l2', penalty='l1', C=1000, dual=False, tol=1e-3)) clf_lSVC = LinearSVC(loss='l2', penalty='l2', C=1000, dual=False, tol=1e-3) clf_SVC = SVC(C=1000) # empty ndarrays for predication results z_kn z_bNB = np.array([], dtype=np.int32) z_mNB = np.array([], dtype=np.int32) z_kNN = np.array([], dtype=np.int32) z_ridge = np.array([], dtype=np.int32) z_SGD = np.array([], dtype=np.int32) z_lSVC = np.array([], dtype=np.int32) z_SVC = np.array([], dtype=np.int32) # Best implementation... Too confusion for now... # z_m = []
def find_best_sgd(**params):
parameters = {
'alpha': [0.0001, 0.0005, 0.001],
'rho': [0.80, 0.85, 0.95],
}
return GridSearchCV(SGDClassifier(**params), parameters)
"Your information to our service providers. We use service providers who help us to provide you with our services. We give relevant persons working for some of these providers access to your information, but only to the extent necessary for them to perform their services for us. We also implement reasonable contractual and technical protections to ensure the confidentiality of your personal information and data is maintained, used only for the provision of their services to us, and handled in accordance with this privacy policy. Examples of service providers include payment processors, email service providers, and web traffic analytics tools", "Some Microsoft sites allow you to choose to share your personal information with select Microsoft partners so that they can contact you about their products, services or offers. Other sites, such as MSN instead may give you a separate choice as to whether you wish to receive communications from Microsoft about a partner's particular offering (without transferring your personal information to the third party). See the Communication Preferences section below for more information.", ] X_new = vectorizer.transform(docs_new) # Train classifiers print "Training Classifiers..." t0 = time() clf_nb = MultinomialNB() clf_lsvc = LinearSVC(loss='l2', penalty='l2', C=1000, dual=False, tol=1e-3) clf_svc = SVC(C=1024, kernel='rbf', degree=3, gamma=0.001, probability=True) clf_rdg = RidgeClassifier(tol=1e-1) clf_sgd = SGDClassifier(alpha=.0001, n_iter=50, penalty="l2") # Logistic regression requires OneVsRestClassifier which hides # its methods such as decision_function # It will require extra implementation efforts to use it as a candidate # for multilabel classification # clf_lgr = OneVsRestClassifier(LogisticRegression(C=1000,penalty='l1')) # kNN does not have decision function due to its nature # clf_knn = KNeighborsClassifier(n_neighbors=13) # train clf_nb.fit(X, y) clf_lsvc.fit(X, y) clf_rdg.fit(X, y) clf_svc.fit(X, y) clf_sgd.fit(X, y)
