def training_dbn(train_dataset, n_targets=2, learn_rates=0.3, learn_rate_decays=0.9, epochs=1000, n_hidden_layers=5, n_hidden_layer_nodes=100,
verbose=True):
layers = np.ones(n_hidden_layers, dtype=int) * n_hidden_layer_nodes
print(layers.tolist())
X_train, y_train = train_dataset
ff = [X_train.shape[1]]
ff.extend(layers.tolist())
ff.append(n_targets)
# Create the dbn
clf = DBN(
ff,
learn_rates=learn_rates,
learn_rate_decays=learn_rate_decays,
epochs=epochs,
dropouts=0.1,
verbose=verbose)
# Counting time for training
start = time.time()
clf.fit(X_train, y_train) # training
end = time.time()
exec_time = end - start
print('Exec time was {} secs'.format(exec_time))
return clf, exec_time
def benchmark(k, epochs):
print("*" * 80)
print("k: %d, epochs: %d\n" % (k, epochs))
#select = SelectKBest(score_func=chi2, k=k)
select = TruncatedSVD(n_components=k)
X_train_trunc = select.fit_transform(X_train, Y_train)
X_test_trunc = select.transform(X_test)
print('done truncating')
clf = DBN([X_train_trunc.shape[1], k, 4], learn_rates=0.3, learn_rate_decays=0.9, epochs=epochs, verbose=1)
clf.fit(X_train_trunc, Y_train)
pred = clf.predict(X_test_trunc)
if CREATE_SUBMISSION:
X_submit_trunc = select.transform(X_submit)
pred_submit = clf.predict(X_submit_trunc)
dump_csv(pred_submit, k, epochs)
score = metrics.f1_score(Y_test, pred)
print("f1-score: %0.3f" % score)
print("classification report:")
print(metrics.classification_report(Y_test, pred))
print("confusion matrix:")
print(metrics.confusion_matrix(Y_test, pred))
def train_model(data_set_path='/home/devin.fisher/Kingdoms/treadstone/_samples/still_data/still_training_data.pkl'):
# data_set = None
with open(data_set_path, 'rb') as f:
data_set = pickle.load(f)
# with open('/home/devin.fisher/Kingdoms/lol/still_training_data2.pkl', 'rb') as f:
# data_set = pickle.load(f)
# (train_x, test_x, train_y, test_y) = train_test_split(data_set['data'], data_set['target'], test_size=0.1)
train_x = data_set['data']
test_x = data_set['data']
train_y = data_set['target']
test_y = data_set['target']
dbn = DBN(
[-1, 300, -1],
learn_rates=0.3,
learn_rate_decays=0.9,
epochs=60,
verbose=1)
dbn.fit(train_x, train_y)
joblib.dump(dbn, 'digit_model.pkl', compress=9)
# dbn = joblib.load('digit_model.pkl')
# compute the predictions for the test data and show a classification report
preds = dbn.predict(test_x)
print classification_report(test_y, preds)
def main():
data_id = 'B'
data_path = '/broad/compbio/maxwshen/data/1-MAKETRAINTEST/complete/golf/'
print 'train...', datetime.datetime.now()
train_set = readin(data_id, 'train', data_path)
print 'valid...', datetime.datetime.now()
valid_set = readin(data_id, 'valid', data_path)
print 'test...', datetime.datetime.now()
test_set = readin(data_id, 'test', data_path)
# Input to 300 node RBM to 2 node output
dbn = DBN( \
[xtrain.shape[1], 300, 2], \
learn_rates = 5, \
learn_rate_decays = 0.9, \
epochs = 31, \
verbose = 1)
dbn.fit(dat_train, y_train)
preds = dbn.predict(dat_test)
print classification_report(y_test, preds)
out_fn = 'dbn.pickle'
with open(out_fn, 'w') as f:
pickle.dump(dbn, out_fn)
return
def train(X, Y, alphabet):
model = DBN([13, 1000, len(alphabet)],
learn_rates=0.3,
learn_rate_decays=0.9,
epochs=10,
verbose=1,)
model.fit(X, Y)
return model
def run():
X_train, Y_train = load_training_data()
X_train, Y_train = rotate_dataset(X_train, Y_train, 8)
X_train, Y_train = nudge_dataset(X_train, Y_train)
n_features = X_train.shape[1]
n_classes = 10
classifier = DBN([n_features, 8000, n_classes],
learn_rates=0.4, learn_rate_decays=0.9 ,epochs=75, verbose=1)
classifier.fit(X_train, Y_train)
test_data = get_test_data_set()
predictions = classifier.predict(test_data)
write_predictions_to_csv(predictions)
def __init__(self):
# images_train=data_train[:,1:]
# trainX, _trainX, trainY, _trainY = train_test_split(images_train/255.,values_train,test_size=0.5)
# #load test.csv
# test = pd.read_csv("data/test.csv")
# data_test=test.as_matrix()
# testX, _testX = train_test_split(data_test/255.,test_size=0.99)
# Random Forest
# self.clf = RandomForestClassifier()
# Stochastic Gradient Descent
# self.clf = SGDClassifier()
# Support Vector Machine
# self.clf = LinearSVC()
# Nearest Neighbors
# self.clf = KNeighborsClassifier(n_neighbors=13)
train = pd.read_csv("data/train.csv")
data_train=train.as_matrix()
values_train=data_train[:,0]
images_train=data_train[:,1:]
trainX, _trainX, trainY, _trainY = train_test_split(images_train/255.,values_train,test_size=0.995)
# Neural Network
self.clf = DBN([trainX.shape[1], 300, 10],learn_rates=0.3,learn_rate_decays=0.9,epochs=10,verbose = 1)
#Training
self.clf.fit(trainX, trainY)
pass
def train_clf(dim, X, y, classificator):
print("Training for {} classes".format(dim[2]))
if classificator == "DBN":
clf = DBN(dim,
learn_rates=dbn_learn_rates,
learn_rate_decays=dbn_learn_rate_decays,
epochs=dbn_epochs,
minibatch_size=dbn_minibatch_size,
verbose=dbn_verbose,
dropouts=dbn_dropouts
)
elif classificator == "GaussianNB":
clf = GaussianNB()
clf.fit(X, y)
return clf
def train_dbn_dataset(dataset, x_test, y_test, alpha, nhidden, epochs, batch_size, noises=[]):
from nolearn.dbn import DBN
num_classes = len(set(y_test))
print "Number of classes", num_classes
x_train, y_train = dataset
dbn_model = DBN([x_train.shape[1], nhidden, num_classes],
learn_rates = alpha,
learn_rate_decays = 0.9,
epochs = epochs,
verbose = 1,
nesterov=False,
minibatch_size=batch_size,
noises = noises)
dbn_model.fit(x_train, y_train)
from sklearn.metrics import classification_report, accuracy_score
y_true, y_pred = y_test, dbn_model.predict(x_test) # Get our predictions
print(classification_report(y_true, y_pred)) # Classification on each digit
print(roc_auc_score(y_true, y_pred)) # Classification on each digit
return y_pred, roc_auc_score(y_true, y_pred)
def dbn_clf(X, y, hidden_sizes=[300], num_epochs=10):
""" deep belief network """
Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, test_size=0.25, random_state=0)
output_categories = np.load(os.path.join(loaddir,'submit_col_name.npy'))
print('Start training Neural Network...')
dbn = DBN(
[Xtrain.shape[1]] + hidden_sizes + [len(output_categories)],
learn_rates = 0.3,
learn_rate_decays = 0.9,
epochs = num_epochs,
verbose = 1)
dbn.fit(Xtrain, ytrain)
ypred = dbn.predict_proba(Xtest)
score = log_loss(ytest, ypred)
print('Log loss = {}'.format(score))
return dbn, score
def test(self):
#iris = datasets.load_iris()
#X, y = iris.data, iris.target
X, y = self.dataMat,self.labelMat
X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=0.6, random_state=12)
#clf = RandomForestClassifier(max_depth=6,min_samples_split=9,min_samples_leaf=15,n_estimators=5)
#clf = DBN([X.shape[1], 24, 2],scales=0.5,learn_rates=0.02,learn_rate_decays = 0.95, learn_rate_minimums =0.001,epochs=500,l2_costs = 0.02*0.031, dropouts=0.2,verbose=0)
#cvnum = ShuffleSplit(2013,n_iter=10,test_size=0.6,train_size=0.4,random_state=0)
for scal in arange(4.5, 5.0, 0.5):
print "**************************************************************"
print "DBN scal=",scal
clf = DBN([X.shape[1], 24,48, 2],scales=0.5,learn_rates=0.01,learn_rate_decays = 0.95, learn_rate_minimums =0.001,epochs=50,l2_costs = 0.02*0.001, dropouts=0.0,verbose=0)
clf.fit(X_train, y_train);
scores = cross_val_score(clf,X,y,cv=3,scoring='roc_auc')
y_pred = clf.predict(X_test);
y_predprob = clf.predict_proba(X_test);
prf=precision_recall_fscore_support(y_test, y_pred, average='binary')
print ("Accuracy: %0.5f (+/- %0.5f)" % (scores.mean(), scores.std() * 2))
print classification_report(y_test,y_pred)
print 'The accuracy is: ', accuracy_score(y_test,y_pred)
print 'The log loss is:', log_loss(y_test, y_predprob)
print 'The ROC score is:', roc_auc_score(y_test,y_predprob[:,1])
def main():
data_fn = "/home/ec2-user/Kellis/data/bravo.formatted/dat.all.txt"
blacklist_fn = "/home/ec2-user/Kellis/data/bravo.formatted/dat.blacklist.txt"
y_fn = "/home/ec2-user/Kellis/data/bravo.formatted/dat.y.txt"
data = read_delimited_txt(data_fn, "\t")
blacklist = read_delimited_txt(blacklist_fn, "\t")
y = read_delimited_txt(y_fn, "\t")
# Get names and remove the first element of each row which is the row number
names = data[0]
data = data[1:]
for i in range(len(data)):
data[i] = data[i][1:]
y = y[1:]
for i in range(len(y)):
y[i] = y[i][-1]
y = convert_y_binary(y)
# Normalizes column-wise so all values are between 0 and 1
data = normalize_0_1(data)
# Split into training, testing
xtrain, xtest, ytrain, ytest = train_test_split(data, y, test_size=0.2, random_state=1)
# Input to 300 node RBM to 2 node output
dbn = DBN([xtrain.shape[1], 300, 2], learn_rates=5, learn_rate_decays=0.9, epochs=501, verbose=1)
dbn.fit(xtrain, ytrain)
preds = dbn.predict(xtest)
print classification_report(ytest, preds)
out_fn = "dbn.pickle"
with open(out_fn, "w") as f:
pickle.dump(dbn, out_fn)
return
def runOfflineML(y, X, classifiers, savemodel=False):
X_train, X_test, y_train, y_test = train_test_split(X, y.astype("int0"), test_size=0.20, random_state=0)
data = dict(x_train=X_train, x_test=X_test, y_train=y_train, y_test=y_test)
cls_stats = initClsStats(classifiers)
for cls_name, cls in classifiers.items():
cls_stats[cls_name]["n_train"] = data["x_train"].shape[0]
cls_stats[cls_name]["n_test"] = data["x_test"].shape[0]
cls_stats[cls_name]["n_features"] = data["x_train"].shape[1]
tick = time.time()
if cls_name == "DBN":
data = dataNormalise(data)
clf = DBN([data["x_train"].shape[1], 300, 2], learn_rates=0.3, learn_rate_decays=0.9, epochs=10, verbose=1)
clf.fit(data["x_train"], data["y_train"])
else:
clf = classifiers[cls_name].fit(data["x_train"], data["y_train"])
if savemodel:
pickle.dump(clf, open(cls_name + ".dat", "w"))
clf = pickle.load(open(cls_name + ".dat", "r"))
cls_stats[cls_name]["training_time"] += time.time() - tick
# check the accuracy on the training set
tick = time.time()
predicted = clf.predict(data["x_test"])
cls_stats[cls_name]["testing_time"] += time.time() - tick
acc = metrics.accuracy_score(data["y_test"], predicted)
cls_stats[cls_name]["accuracy"] = acc
print cls_name, "accuracy is: " + str(acc)
# auc = metrics.roc_auc_score(data['y_test'], probs[:, 1])
conf_matrix = metrics.confusion_matrix(data["y_test"], predicted)
cls_stats[cls_name]["conf_matrix"] = conf_matrix
# print conf_matrix
precision, recall, fscore, support = metrics.precision_recall_fscore_support(data["y_test"], predicted)
cls_stats[cls_name]["precision"] = precision
cls_stats[cls_name]["recall"] = recall
cls_stats[cls_name]["fscore"] = fscore
cls_stats[cls_name]["support"] = support
return cls_stats
def fit(self, X, y, X_pretrain=None):
from nolearn.dbn import DBN
if y.ndim == 2:
n_outputs = y.shape[1]
else:
y = y[:, np.newaxis]
n_outputs = 1
params = dict(self.__dict__)
from gdbn.activationFunctions import Linear
params['output_act_funct'] = Linear()
n_units = params.pop('n_units')
n_hidden_layers = params.pop('n_hidden_layers')
if isinstance(n_units, int):
units = [n_units] * n_hidden_layers
else:
units = n_units
units = [X.shape[1]] + units + [n_outputs]
self.dbn = DBN(units, **params)
print X.shape
self.dbn.fit(X, y, X_pretrain=X_pretrain)
class DBNRegressor(BaseEstimator, RegressorMixin):
def __init__(self, n_hidden_layers=2, n_units=1000, epochs=100,
epochs_pretrain=0, scales=0.05,
real_valued_vis=True,
use_re_lu=False,
uniforms=False,
learn_rates_pretrain=0.1,
learn_rates=0.1,
learn_rate_decays=1.0,
learn_rate_minimums=0.0,
momentum=0.9,
momentum_pretrain=0.9,
l2_costs=0.0001,
l2_costs_pretrain=0.0001,
dropouts=None,
minibatch_size=64,
verbose=2,
fine_tune_callback=None,
nest_compare=True,
nest_compare_pretrain=None,
fan_outs=None,
nesterov=False,
):
self.n_hidden_layers = n_hidden_layers
self.n_units = n_units
self.epochs = epochs
self.epochs_pretrain = epochs_pretrain
self.learn_rates_pretrain = learn_rates_pretrain
self.learn_rates = learn_rates
self.learn_rate_decays = learn_rate_decays
self.learn_rate_minimums = learn_rate_minimums
self.l2_costs_pretrain = l2_costs_pretrain
self.l2_costs = l2_costs
self.momentum = momentum
self.momentum_pretrain = momentum_pretrain
self.verbose = verbose
self.real_valued_vis = real_valued_vis
self.use_re_lu = use_re_lu
self.scales = scales
self.minibatch_size = minibatch_size
if dropouts is None:
dropouts = [0.2] + [0.5] * n_hidden_layers
self.dropouts = dropouts
self.fine_tune_callback = fine_tune_callback
self.nest_compare = nest_compare
self.nest_compare_pretrain = nest_compare_pretrain
self.fan_outs = fan_outs
self.nesterov = nesterov
def fit(self, X, y, X_pretrain=None):
from nolearn.dbn import DBN
if y.ndim == 2:
n_outputs = y.shape[1]
else:
y = y[:, np.newaxis]
n_outputs = 1
params = dict(self.__dict__)
from gdbn.activationFunctions import Linear
params['output_act_funct'] = Linear()
n_units = params.pop('n_units')
n_hidden_layers = params.pop('n_hidden_layers')
if isinstance(n_units, int):
units = [n_units] * n_hidden_layers
else:
units = n_units
units = [X.shape[1]] + units + [n_outputs]
self.dbn = DBN(units, **params)
print X.shape
self.dbn.fit(X, y, X_pretrain=X_pretrain)
def predict(self, X):
return self.dbn.chunked_decision_function(X)
:class:`~gdbn.activationFunctions.Sigmoid`,
:class:`~.gdbn.activationFunctions.Linear`,
:class:`~.gdbn.activationFunctions.Softmax`
from the
:mod:`gdbn.activationFunctions`
module. Defaults to
:class:`~.gdbn.activationFunctions.Softmax`.
"""
dbn = DBN(
layer_sizes = [trainX.shape[1], 800, 10], # trainX.shape[1] is the input layer, 10 is output layer
# 300 is the hidden layer
# use -1 as last layer if one does not know how many labels
# are there
output_act_funct = "Softmax",
dropouts = 0.5,
use_re_lu=True,
l2_costs=0.0001,
learn_rates = 0.3,
learn_rate_decays = 0.9,
epochs = 10,
loss_funct = None, # if not specified, default is the count of percentage or wrong labels, built in function
verbose = 1)
##### Below is the trick for changing score function to evaluate the accuracy. The original program
# does not have other options except for pure compare % of accurate outputs,
# here one may create a function of his/her own.
import new
def _score(self, X, y):
outputs = self.predict_proba(X)
def do_operation_(X_train,X_test,y_train,y_test,l_r,d_r): clf = DBN([np.shape(X_train)[1],300,10],learn_rates = l_r,learn_rate_decays = d_r,epochs = 30,verbose = 1 ) clf.fit(X_train,y_train) y_test,y_pred = y_test, clf.predict(X_test) result = np.sum(y_test == y_pred) return (result,l_r,d_r)
def main():
"""."""
from sklearn.cross_validation import KFold
set_verbosity(3)
overlap_df = get_data("./vectors/google_overlap.csv")
#overlap_df = get_data("./vectors/freebase_overlap.csv")
overlap_df = overlap_df[overlap_df.NER != 'O']
overlap_df = overlap_df[overlap_df.NER != 'I-FAC']
overlap_df = overlap_df[overlap_df.NER != 'B-FAC']
overlap_df = overlap_df[overlap_df.NER != 'I-LOC']
overlap_df = overlap_df[overlap_df.NER != 'B-LOC']
overlap_df = overlap_df[overlap_df.NER != 'I-WEA']
overlap_df = overlap_df[overlap_df.NER != 'B-WEA']
overlap_df = overlap_df[overlap_df.NER != 'I-VEH']
overlap_df = overlap_df[overlap_df.NER != 'B-VEH']
overlap_df = overlap_df[overlap_df.NER != 'I-TTL']
overlap_df = overlap_df[overlap_df.NER != 'B-TTL']
#overlap_df = overlap_df.groupby("NER").filter(lambda x: len(x) > 50)
label_map, labels = map_labels(overlap_df)
X, y = parse_data(overlap_df, label_map)
trainX, testX, trainY, testY = train_test_split(X, y, test_size=0.10)
count, n_folds, scores = 0, 20, []
logging.info("Beginning Cross Validation with " + str(n_folds) + " folds")
kf = KFold(len(trainX), n_folds=n_folds)
lrs = list(np.linspace(0.1, 0.4, num=n_folds))
for train, test in kf:
logging.debug("TRAIN:" + str(len(train)) + " TEST:" + str(len(test)))
trainX_fold, validX_fold = trainX[train], trainX[test]
trainY_fold, validY_fold = trainY[train], trainY[test]
google_topology = [trainX_fold.shape[1], 300, 200, 100, len(labels)]
#freebase_topology = [trainX_fold.shape[1], 750, 500, 250, len(labels)]
dbn = DBN(
#freebase_topology,
google_topology,
learn_rates=float(lrs[count]),
learn_rate_decays=0.9,
epochs=50,
verbose=0)
dbn.fit(trainX_fold, trainY_fold)
score = dbn.score(validX_fold, validY_fold)
scores.append((score, float(lrs[count])))
count += 1
logging.info(
"Learning rate: " + str(float(lrs[count-1])) + " score:" + \
str(score) + " " + str(float(count)/float(n_folds) * 100) + "% done")
best_lr = max(scores, key=lambda x: x[0])[1]
logging.info("Best CV score: " + str(best_lr))
google_topology = [trainX.shape[1], 300, 200, 100, len(labels)]
#freebase_topology = [trainX.shape[1], 750, 500, 250, len(labels)]
dbn = DBN(
#freebase_topology,
google_topology,
learn_rates=best_lr,
learn_rate_decays=0.9,
epochs=100,
verbose=1)
dbn.fit(trainX, trainY)
preds = dbn.predict(testX)
print classification_report(testY, preds)
#model_and_data = (dbn, label_map)
#dump_model(model_and_data, './google_model.pkl')
#'''
'''
import numpy as np
from sklearn import cross_validation
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import LinearSVC
from sklearn.linear_model import SGDClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score
from sklearn import datasets
from sklearn import preprocessing
from skimage.feature import hog
from nolearn.dbn import DBN
import timeit
train = pd.read_csv("../train.csv")
features = train.columns[1:]
X = train[features]
y = train['label']
X_train, X_test, y_train, y_test = cross_validation.train_test_split(
X / 255., y, test_size=0.1, random_state=0)
clf_nn = DBN([X_train.shape[1], 300, 10],
learn_rates=0.3,
learn_rate_decays=0.9,
epochs=15)
training_x = X_train.as_matrix()
training_y = y_train.as_matrix()
testing_x = X_test.as_matrix()
testing_y = y_test.as_matrix()
clf_nn.fit(training_x, training_y)
acc_nn = clf_nn.score(testing_x, testing_y)
print "neural network accuracy: ", acc_nn
x_train, x_test, y_train, y_test, x, y = train_test_prep()
#dbn_model = DBN([x_train.shape[1], 1500, 1500, 2],
# learn_rates = 0.01,
# learn_rate_decays = 0.9,
# epochs = 1000,
# verbose = 3)
dbn_model = DBN([x_train.shape[1], 5000, 2500, 1250, 500],
#dropouts=0.01,
output_act_funct=activationFunctions.Sigmoid(),
learn_rates=0.01,
learn_rates_pretrain=0.001,
# minibatch_size=9,
# learn_rate_decays=0.9,
# learn_rate_minimums=0.0001,
epochs_pretrain=500,
epochs=500,
# momentum= self.momentum,
# real_valued_vis=True,
# use_re_lu=True,
verbose=2)
dbn_model.fit(x_train, y_train)
y_true, y_pred = y_test, dbn_model.predict(x_test) # Get our predictions
print(classification_report(y_true, y_pred)) # Classification on each digit
print y_true
print y_pred
def main(input_file):
batch_size = 128
nb_classes = 62 # A-Z, a-z and 0-9
nb_epoch = 2
# Input image dimensions
img_rows, img_cols = 32, 32
# Path of data files
path = input_file
def convert_(Y):
alpha = string.letters
dig = string.digits
alphaList = []
for elem in (alpha + dig):
alphaList.append(elem)
list_ = []
for elem in Y:
for i in range(0, elem.shape[0]):
if elem[i] == 1:
list_.append(i)
list_ = np.asarray(list_)
return list_
# Load the preprocessed data and labels
X_train_all = np.load(path + "/trainPreproc_" + str(img_rows) + "_" +
str(img_cols) + ".npy")
Y_train_all = np.load(path + "/labelsPreproc.npy")
X_train, X_val, Y_train, Y_val = \
train_test_split(X_train_all, Y_train_all, test_size=0.25, stratify=np.argmax(Y_train_all, axis=1))
print X_train.shape
labels = convert_(Y_train)
validation = convert_(Y_val)
X_train = X_train.reshape(
(X_train.shape[0], X_train.shape[2] * X_train.shape[3]))
X_val = X_val.reshape((X_val.shape[0], X_val.shape[2] * X_val.shape[3]))
print 'Training and Testing...'
clf_rf = RandomForestClassifier()
clf_rf.fit(X_train, labels)
y_pred_rf = clf_rf.predict(X_val)
acD_rf = accuracy_score(validation, y_pred_rf)
print "random forest accuracy: ", acD_rf
clf_sgd = SGDClassifier()
clf_sgd.fit(X_train, labels)
y_pred_sgd = clf_sgd.predict(X_val)
acD_sgd = accuracy_score(validation, y_pred_sgd)
print "stochastic gradient descent accuracy: ", acD_sgd
clf_svm = LinearSVC()
clf_svm.fit(X_train, labels)
y_pred_svm = clf_svm.predict(X_val)
acD_svm = accuracy_score(validation, y_pred_svm)
print "Linear SVM accuracy: ", acD_svm
clf_knn = KNeighborsClassifier()
clf_knn.fit(X_train, labels)
y_pred_knn = clf_knn.predict(X_val)
acD_knn = accuracy_score(validation, y_pred_knn)
print "nearest neighbors accuracy: ", acD_knn
clf_nn = DBN([X_train.shape[1], 300, 62],
learn_rates=0.0240,
learn_rate_decays=0.9,
epochs=130)
clf_nn.fit(X_train, labels)
acD_nn = clf_nn.score(X_val, validation)
print "neural network accuracy: ", acD_nn
clf = MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)
clf.fit(X_train, labels)
acD_nn = clf.score(X_val, validation)
print "naive bayes: ", acD_nn
clf = BernoulliNB(alpha=1.0,
binarize=0.0,
class_prior=None,
fit_prior=True)
clf.fit(X_train, labels)
acD_nn = clf.score(X_val, validation)
print "bernulli naive bayes: ", acD_nn
def rotate_dataset(X):
XX = np.zeros(X.shape)
for index in range(X.shape[0]):
angle = np.random.randint(-7, 7)
XX[index, :] = nd.rotate(np.reshape(X[index, :], ((28, 28))),
angle,
reshape=False).ravel()
return XX
# Load Data
mnist = pd.read_csv("data_5k.csv")
#mnist = mnist[:50]
y_train = mnist['label'].values
X_train = mnist.loc[:, 'pixel0':].values
X_test = pd.read_csv("test.csv").values
X_test = X_test[:2000]
X_train = np.asarray(X_train / 255.0, 'float32')
X_test = np.asarray(X_test / 255.0, 'float32')
#X_train, y_train = nudge_dataset(X_train, y_train)
#X_train = rotate_dataset(X_train)
clf = DBN([X_train.shape[1], 350, 10],\
learn_rates=0.3,\
learn_rate_decays=0.95,\
learn_rates_pretrain=0.005,\
epochs=120,\
verbose=1)
clf.fit(X_train, y_train)
subm = pd.read_csv("rf_benchmark.csv")
subm.Label = clf.predict(X_test)
subm.to_csv("result.csv", index_label='ImageId', col=['Label'], index=False)
def train(args):
start = time.time()
for clfChoice in clfChoices:
print("Loading embeddings.")
fname = "{}/labels.csv".format(args.workDir)
labels = pd.read_csv(fname, header=None).as_matrix()[:, 1]
labels = map(itemgetter(1),
map(os.path.split, map(os.path.dirname,
labels))) # Get the directory.
fname = "{}/reps.csv".format(args.workDir)
embeddings = pd.read_csv(fname, header=None).as_matrix()
le = LabelEncoder().fit(labels)
labelsNum = le.transform(labels)
nClasses = len(le.classes_)
print("Training for {} classes.".format(nClasses))
if clfChoice == 'LinearSvm':
clf = SVC(C=1, kernel='linear', probability=True)
elif clfChoice == 'GMM': # Doesn't work best
clf = GMM(n_components=nClasses)
# ref:
# http://scikit-learn.org/stable/auto_examples/classification/plot_classifier_comparison.html#example-classification-plot-classifier-comparison-py
elif clfChoice == 'RadialSvm': # Radial Basis Function kernel
# works better with C = 1 and gamma = 2
clf = SVC(C=1, kernel='rbf', probability=True, gamma=2)
elif clfChoice == 'DecisionTree': # Doesn't work best
clf = DecisionTreeClassifier(max_depth=20)
elif clfChoice == 'GaussianNB':
clf = GaussianNB()
# ref: https://jessesw.com/Deep-Learning/
elif clfChoice == 'DBN':
if args.verbose:
verbose = 1
else:
verbose = 0
clf = DBN(
[embeddings.shape[1], 500, labelsNum[-1:][0] + 1
], # i/p nodes, hidden nodes, o/p nodes
learn_rates=0.3,
# Smaller steps mean a possibly more accurate result, but the
# training will take longer
learn_rate_decays=0.9,
# a factor the initial learning rate will be multiplied by
# after each iteration of the training
epochs=300, # no of iternation
# dropouts = 0.25, # Express the percentage of nodes that
# will be randomly dropped as a decimal.
verbose=verbose)
if args.ldaDim > 0:
clf_final = clf
clf = Pipeline([('lda', LDA(n_components=args.ldaDim)),
('clf', clf_final)])
clf.fit(embeddings, labelsNum)
fName = os.path.join(args.workDir, clfChoice + ".pkl")
print("Saving classifier to '{}'".format(fName))
with open(fName, 'w') as f:
pickle.dump((le, clf), f)
if args.verbose:
print("Training and saving the classifiers took {} seconds.".format(
time.time() - start))
def classifier(args,args_mode,dataset,sess):
# Check that there are at least one training image per class
for cls in dataset:
#print(cls.name,'@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@')
if(len(cls.image_paths)<1):
print(cls.image_paths,"@@@@@@@@@@@@@@@@@@@@@@@")
assert (len(cls.image_paths) > 0, 'There must be at least one image for each class in the dataset')
paths, labels,class_labels = get_image_paths_and_labels(dataset)
print('Number of classes: %d' % len(dataset))
print('Number of images: %d' % len(paths))
# Get input and output tensors
images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0")
phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")
embedding_size = embeddings.get_shape()[1]
# Run forward pass to calculate embeddings
print('Calculating features for images')
nrof_images = len(paths)
nrof_batches_per_epoch = int(math.ceil(1.0 * nrof_images / args.batch_size))
emb_array = np.zeros((nrof_images, embedding_size))
for i in range(nrof_batches_per_epoch):
start_index = i * args.batch_size
end_index = min((i + 1) * args.batch_size, nrof_images)
paths_batch = paths[start_index:end_index]
images = facenet.load_data(paths_batch, False, False, args.image_size)
feed_dict = {images_placeholder: images, phase_train_placeholder: False}
emb_array[start_index:end_index, :] = sess.run(embeddings, feed_dict=feed_dict)
classifier_filename_exp = os.path.expanduser(args.classifier_filename)
if (args_mode == 'TRAIN'):
# Train classifier
print('Training classifier+++++++++++++++++++++++++',args.classifier)
if args.classifier == 'LinearSvm':
# clf = SVC(C=1, kernel='linear', probability=True)
model = SVC(kernel='linear', probability=True)
elif args.classifier == 'GridSearchSvm':
print("""
Warning: In our experiences, using a grid search over SVM hyper-parameters only
gives marginally better performance than a linear SVM with C=1 and
is not worth the extra computations of performing a grid search.
""")
param_grid = [
{'C': [1, 10, 100, 1000],
'kernel': ['linear']},
{'C': [1, 10, 100, 1000],
'gamma': [0.001, 0.0001],
'kernel': ['rbf']}
]
model = GridSearchCV(SVC(C=1, probability=True), param_grid, cv=5)
elif args.classifier == 'GMM': # Doesn't work best
model = GMM(n_components=nClasses)
# ref:
# http://scikit-learn.org/stable/auto_examples/classification/plot_classifier_comparison.html#example-classification-plot-classifier-comparison-py
elif args.classifier == 'RadialSvm': # Radial Basis Function kernel
# works better with C = 1 and gamma = 2
model = SVC(C=1, kernel='rbf', probability=True, gamma=2)
elif args.classifier == 'DecisionTree': # Doesn't work best
model = DecisionTreeClassifier(max_depth=20)
elif args.classifier == 'GaussianNB':
model = GaussianNB()
# ref: https://jessesw.com/Deep-Learning/
elif args.classifier == 'DBN':
from nolearn.dbn import DBN
model = DBN([embeddings.shape[1], 500, labelsNum[-1:][0] + 1], # i/p nodes, hidden nodes, o/p nodes
learn_rates=0.3,
# Smaller steps mean a possibly more accurate result, but the
# training will take longer
learn_rate_decays=0.9,
# a factor the initial learning rate will be multiplied by
# after each iteration of the training
epochs=300, # no of iternation
# dropouts = 0.25, # Express the percentage of nodes that
# will be randomly dropped as a decimal.
verbose=1)
elif args.classifier == 'KNN':
model = KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',
metric_params=None, n_jobs=1, n_neighbors=5, p=2,
weights='uniform')
model.fit(emb_array, labels)
# Create a list of class names
class_names = [cls.name.replace('_', ' ') for cls in dataset]
# Saving classifier model
with open(classifier_filename_exp, 'wb') as outfile:
pickle.dump((model, class_names), outfile)
print('Saved classifier model to file "%s"' % classifier_filename_exp)
elif (args_mode == 'CLASSIFY'):
# Classify images
print('Testing classifier~~~~~~~~~~~~~~~~~~~~~~~~')
with open(classifier_filename_exp, 'rb') as infile:
(model, class_names) = pickle.load(infile)
predictions = np.zeros((nrof_images, len(class_names)))
print('Loaded classifier model from file "%s"' % classifier_filename_exp)
correctPrediction = 0
inCorrectPrediction = 0
sumConfidence = 0.0
correctConfidence = 0.0
inCorrectConfidence = 0.0
'''
batch_size =args.batch_size
#batch_size = 1
for i in range(nrof_batches_per_epoch):
start_index = i * batch_size
end_index = min((i + 1) * batch_size, nrof_images)
starttime = time.time()
mini_emb_array = emb_array[start_index:end_index, :]
predictions[start_index:end_index, :] = model.predict_proba(mini_emb_array)
print("start_index:{} end_index:{} time:{}".format(start_index, end_index, time.time() - starttime))
'''
predictions = model.predict_proba(emb_array)
best_class_indices = np.argmax(predictions, axis=1)
best_class_probabilities = predictions[np.arange(len(best_class_indices)), best_class_indices]
results = {'name': [], 'bestname': [], 'probabilities': []}
for i in range(len(best_class_indices)):
#print(len(class_names))
#print(i,len(labels),labels[i])
#print(i,len(best_class_indices),best_class_indices[i])
print('%4d %s:%s: %.3f' % (
i, class_labels[i], class_names[best_class_indices[i]], best_class_probabilities[i]))
results['name'].append(class_labels[i])
results['bestname'].append(class_names[best_class_indices[i]])
results['probabilities'].append(best_class_probabilities[i])
sumConfidence += best_class_probabilities[i]
if (class_labels[i] == class_names[best_class_indices[i]]):
correctPrediction += 1
correctConfidence += best_class_probabilities[i]
else:
inCorrectPrediction += 1
inCorrectConfidence += best_class_probabilities[i]
#accuracy = np.mean(np.equal(best_class_indices, labels))
accuracy = float(correctPrediction) / (correctPrediction + inCorrectPrediction)
Avg_Confidence = float(sumConfidence) / (correctPrediction + inCorrectPrediction)
Avg_correctConfidence = float(correctConfidence/correctPrediction)
Avg_inCorrectConfidence = float(inCorrectConfidence / inCorrectPrediction)
results['name'].append('Accuracy:')
results['bestname'].append('Accuracy:')
results['probabilities'].append(accuracy)
dataname = datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')
data_frame = pd.DataFrame(
data={'name': results['name'], 'bestname': results['bestname'], 'probabilities': results['probabilities']})
data_frame.to_csv(args.data_dir + '/results_' + dataname + '.csv')
print("Correct Prediction :" + str(correctPrediction))
print("In-correct Prediction: " + str(inCorrectPrediction))
print('Accuracy: %.3f' % accuracy)
print("Avg Confidence: " + str(Avg_Confidence))
print("Avg CorrectConfidence: " + str(Avg_correctConfidence))
print("Avg inCorrectConfidence: " + str(Avg_inCorrectConfidence))
from sklearn.cross_validation import train_test_split
from sklearn.metrics import classification_report
from sklearn import datasets
from nolearn.dbn import DBN
import numpy as np
print("[INFO] downloading mnist")
dataset = datasets.fetch_mldata("MNIST Original")
(trainData, testData, trainLabels,
testLabels) = train_test_split(dataset.data / 255.0,
dataset.target.astype("int"),
test_size=0.33)
dbn = DBN([trainData.shape[1], 300, 10],
learn_rates=0.3,
learn_rate_decays=0.9,
epochs=10,
verbose=1)
dbn.fit(trainData, trainLabels)
predictions = dbn.predict(testData)
print(classification_report(testLabels, predictions))
print train_inputs.shape
print test_inputs.shape
sys.exit(0)
X = joblib.load('features/X_train_processed.pkl')
Y = np.load('blobs/Y_train.npy')
print("done loading")
CREATE_SUBMISSION = False
if CREATE_SUBMISSION:
X_submit = joblib.load('features/X_submit_processed.pkl')
X = select.transform(X)
X_submit = select.transform(X_submit)
clf = DBN([X.shape[1], 280, 10],
learn_rates=.1,
learn_rate_decays=0.9,
momentum=0.9,
epochs=100,
verbose=1)
clf.fit(X, Y)
print("done fit")
pred = clf.predict(X_submit)
f = open('preds_nn_2304x280x10_100epochs_processed.csv', 'wb')
f.write('Id,Prediction\n')
for i, p in enumerate(pred):
f.write("%d,%d\n" % (i + 1, p))
f.close()
else:
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
test_size=0.1,
random_state=42)
from sklearn.metrics import classification_report
mnist = fetch_mldata('mnist-original')
X_train, X_test, y_train, y_test = train_test_split(
(mnist.data / 255.0).astype(np.float32),
mnist.target.astype(np.int32),
test_size=1.0 / 7.0,
random_state=1234)
classifiers = []
if 'dbn' in sys.argv:
from nolearn.dbn import DBN
clf = DBN([X_train.shape[1], 300, 10],
learn_rates=0.3,
learn_rate_decays=0.9,
epochs=10,
verbose=1)
classifiers.append(('nolearn.dbn', clf))
if 'sknn' in sys.argv:
from sknn.mlp import Classifier, Layer, Convolution
clf = Classifier(
layers=[
# Convolution("Rectifier", channels=10, pool_shape=(2,2), kernel_shape=(3, 3)),
Layer('Rectifier', units=200),
Layer('Softmax')
],
learning_rate=0.01,
learning_rule='momentum',
for pipe in [pipeline_3]:
model_generic_1ofK_clas(pipe,
model_name="rf_1",
model_f=sklearn_prob_model(
RandomForestClassifier(n_estimators=200,
n_jobs=6)),
essays_paths=ALL_ESSAYS,
parallel=False)
for pipe in [pipeline_8]:
model_generic_DBN(pipeline=pipe,
model_name="nn_3",
model_f=DBN([-1, 800, -1],
learn_rates=0.1,
learn_rate_decays=0.9,
use_re_lu=True,
epochs=200,
verbose=0,
dropouts=[0.5, 0.1, 0.0],
momentum=0.9),
essays_paths=ALL_ESSAYS)
for pipe in PIPELINES:
for ntrees in [50, 100, 200, 300]:
xgbparam = {
'max_depth': 6,
'eta': 0.1,
'silent': 1,
'objective': 'binary:logistic',
'nthread': 6
}
xgbparamser = serialize_dict(xgbparam) + "_ntrees=%d" % (ntrees)
def train(self, impostor, negative_samples):
"""
impostor: just string rep of the impostor
negative_samples: Set of impostor examples - which we label 0. This does not include
examples from impostor.
Training examples remain the same.
At the end, will update self.dbns and add a new trained dbn for this
impostor.
"""
train_X = np.copy(self.train_X)
samples = np.vstack((train_X, negative_samples))
labels = []
# Add labels: 1 - user, and 0 - impostor.
for i in self.train_X:
labels.append(1)
for i in negative_samples:
labels.append(0)
labels = np.array(labels)
# print("details of train_X AFTER NORM...")
# print("len trainX is = ", len(samples))
# print("train X[0] is ", samples[0])
# print("train X is ", samples)
if self.normalize:
sample = samples[0][1]
self._normalize(samples)
assert samples[0][1] != sample, "normalization not done"
# Nothing disastrous appears to have happened before/after...
# print("details of train_X before...")
# print("len trainX is = ", len(samples))
# print("train X[0] is ", samples[0])
# print("train X is ", samples)
# FIXME: Adds some randomness into the training process - should we
# avoid this?
# samples, labels = unison_shuffled_copies(samples, labels)
file_name = self.gen_pickle_name(impostor, str(len(negative_samples)))
# Training is expensive so let us save and load it if possible.
# FIXME Update this - just want to make sure we are training new guy
# everytime for now...
if os.path.isfile(file_name):
# if False:
with open(file_name, 'rb') as handle:
self.dbns[impostor] = pickle.load(handle)
print("loaded file ", file_name, "from disk")
else:
# let us train this guy and then save it.
self.dbns[impostor] = DBN(
self.shape,
learn_rates=self.learn_rates,
epochs=self.epochs,
learn_rates_pretrain=self.learn_rates_pretrain,
epochs_pretrain=self.epochs_pretrain,
learn_rate_decays=self.learn_rate_decays,
verbose=self.verbose,
minibatches_per_epoch=self.minibatches_per_epoch,
loss_funct=self.loss_funct)
self.dbns[impostor].fit(samples, labels)
with open(file_name, 'w+') as handle:
pickle.dump(self.dbns[impostor],
handle,
protocol=pickle.HIGHEST_PROTOCOL)
print("saved ", file_name, "on disk")
trainX = dataset['data'].T[0:500, 0:] / 255.0
trainY = dataset['label'][0][0:20000]
testX = dataset['data'].T[0:, 0:] / 255.0
testY = dataset['label'][0][0:]
# train the Deep Belief Network with 784 input units (the flattened,
# 28x28 grayscale image), 300 hidden units, 10 output units (one for
# each possible output classification, which are the digits 1-10)
print trainX.shape, trainY.shape
# trainY = np.array(range(2000))
# print type(trainY), trainY
# exit()
print "shape[0]", trainX.shape[1]
dbn = DBN([trainX.shape[1], 500, 3],
learn_rates=0.1,
learn_rate_decays=0.9,
epochs=2500,
verbose=1)
dbn.fit(trainX, trainY)
with open('data.pkl', 'wb') as output:
pickle.dump(dbn, output, pickle.HIGHEST_PROTOCOL)
# # compute the predictions for the test data and show a classification
# # report
# preds = dbn.predict(testX)
# print classification_report(testY, preds)
# randomly select a few of the test instances
# for i in np.random.choice(np.arange(0, len(testY)), size = (500,)):
# # classify the digit
# print testX[i].shape
def train(self, workDir, classifier='LinearSvm'):
print("Loading embeddings.")
fname = "{}/labels.csv".format(workDir)
labels = pd.read_csv(fname, header=None).as_matrix()[:, 1]
labels = map(itemgetter(1),
map(os.path.split, map(os.path.dirname, labels)))
fname = "{}/reps.csv".format(workDir)
embeddings = pd.read_csv(fname, header=None).as_matrix()
le = LabelEncoder().fit(labels)
labelsNum = le.transform(labels)
nClasses = len(le.classes_)
print("Training for {} classes.".format(nClasses))
if classifier == 'LinearSvm':
clf = SVC(C=1, kernel='linear', probability=True)
elif classifier == 'GMM':
clf = GMM(n_components=nClasses)
elif classifier == 'RadialSvm':
clf = SVC(C=1, kernel='rbf', probability=True, gamma=2)
elif classifier == 'DecisionTree':
clf = DecisionTreeClassifier(max_depth=20)
elif classifier == 'GaussianNB':
clf = GaussianNB()
elif classifier == 'DBN':
from nolearn.dbn import DBN
clf = DBN([embeddings.shape[1], 500, labelsNum[-1:][0] + 1],
learn_rates=0.3,
learn_rate_decays=0.9,
epochs=300,
verbose=1)
if self.ldaDim > 0:
clf_final = clf
clf = Pipeline([('lda', LDA(n_components=self.ldaDim)),
('clf', clf_final)])
clf.fit(embeddings, labelsNum)
fName = "{}/classifier.pkl".format(workDir)
print("Saving classifier to '{}'".format(fName))
with open(fName, 'w') as f:
pickle.dump((le, clf), f)
from sklearn.datasets import fetch_mldata
mnist = fetch_mldata('mnist-original')
X_train, X_test, y_train, y_test = train_test_split(
(mnist.data / 255.0).astype(np.float32),
mnist.target.astype(np.int32),
test_size=0.33,
random_state=1234)
classifiers = []
if 'dbn' in sys.argv:
from nolearn.dbn import DBN
clf = DBN([X_train.shape[1], 300, 10],
learn_rates=0.3,
learn_rate_decays=0.9,
epochs=10,
verbose=1)
classifiers.append(('nolearn.dbn', clf))
if 'sknn' in sys.argv:
from sknn import mlp
clf = mlp.Classifier(
layers=[mlp.Layer("Rectifier", units=300),
mlp.Layer("Softmax")],
learning_rate=0.02,
learning_rule='momentum',
batch_size=25,
valid_size=0.0,
n_stable=10,
# duration = end - start
# print "Executing time is %s" %str(duration)
# print m.score(X_test, y_test)
# print "\n"
# print X_train[:2, :]
# print y_train[:2]
# m = MLP()
# m.fit(X_train, y_train, 0.0001, 500)
# p = m.predict(X_test)
# print classification_report(y_test, p)
# h = [i for i in range(1,1000, 200)]
dbn = DBN(
[X_train.shape[1], 200, 3],
learn_rates = 0.0001,
learn_rate_decays = 0.9,
epochs = 10,
verbose = 1
)
dbn.fit(X_train, y_train)
p = dbn.predict(X_test)
a = accuracy_score(y_test, p)
eva = classification_report(y_test, p)
print "Accuracy = %.2f" %a
print eva
train = poly.fit_transform(train)
test = poly.transform(test)
#train = np.hstack((train, poly_train))
#test = np.hstack((test, poly_test))
# encode labels
lbl_enc = LabelEncoder()
labels = lbl_enc.fit_transform(labels)
# set up datasets for cross eval
x_train, x_test, y_train, y_test = train_test_split(train, labels)
# train a DBN classifier
clf = DBN([train.shape[1], 8000, 9],
learn_rates=0.3,
learn_rate_decays=0.9,
epochs=50,
verbose=1) # l2_costs = 0.0001,
clf.fit(x_train, y_train)
# predict on test set
preds = clf.predict_proba(x_test)
# ---------------------- create submission file -----------------------------
#preds = pd.DataFrame(preds, index=sample.id.values, columns=sample.columns[1:])
#preds.to_csv('Preds/dbn_amax.csv', index_label='id')
# ---------------------- cross eval -----------------------------------------
#y_test = label_binary.inverse_transform(y_test)
digits_label = data[:, 0]
digits_image = data[:, 1:]
# digits_label = np.array([i[0] for i in data])
# digits_image = np.array([i[1:]/255.0 for i in data])
digits_image = map(binaryzation, digits_image)
print("file loading ok , prepare to train model")
# object of ANN model.
clf = DBN(
[digits_image.shape[1], 500, 500, 10],
learn_rates=0.01,
learn_rate_decays=0.9,
epochs=20,
verbose=1,
)
# gnb.fit(digits_image[:-testRest],digits_label[:-testRest])
clf.fit(digits_image[:-testRest], digits_label[:-testRest])
#result = gnb.predict(digits_image[-testRest:])
result = clf.predict(digits_image[-testRest:])
print("the err rate : %.2f %%" %
(100 * ((result != digits_label[-testRest:]).sum()) / float(testRest)))
print("the result num %.2f " % (result != digits_label[-testRest:]).sum())
# normalization (recommendable)
trainFeats -= trainFeats.min()
trainFeats /= trainFeats.max()
# Loading test data
print "loading test features and labels ..."
testFeats = pd.read_csv(testFile, delim_whitespace=True, header=None)
testFeats = testFeats.as_matrix()
testLabels = pd.read_csv(testLabelsFile, header=None).values.ravel()
# normalization (recommendable)
testFeats -= testFeats.min()
testFeats /= testFeats.max()
dbn = DBN(
[trainFeats.shape[1], 2000, 7],
learn_rates=0.1,
learn_rate_decays=0.9,
epochs=10,
verbose=1)
dbn.fit(trainFeats, trainLabels)
predictedTest = dbn.predict(testFeats)
# Print classification report
print(classification_report(testLabels, predictedTest))
# Print confusion matrix
print(confusion_matrix(testLabels, predictedTest))
# generate polynomial features
poly = PolynomialFeatures()
train = poly.fit_transform(train)
test = poly.transform(test)
#train = np.hstack((train, poly_train))
#test = np.hstack((test, poly_test))
# encode labels
lbl_enc = LabelEncoder()
labels = lbl_enc.fit_transform(labels)
# set up datasets for cross eval
x_train, x_test, y_train, y_test = train_test_split(train, labels)
# train a DBN classifier
clf = DBN([train.shape[1], 8000, 9], learn_rates = 0.3,
learn_rate_decays = 0.9, epochs = 50, verbose = 1) # l2_costs = 0.0001,
clf.fit(x_train, y_train)
# predict on test set
preds = clf.predict_proba(x_test)
# ---------------------- cross eval -----------------------------------------
#y_test = label_binary.inverse_transform(y_test)
#y_test = LabelEncoder().fit_transform(y_test)
print("Multiclass Log Loss: ", MultiLogLoss(y_test, preds))
import math
import numpy as np
from nolearn.dbn import DBN
from sklearn.cross_validation import train_test_split
from sklearn.decomposition import TruncatedSVD, PCA, RandomizedPCA
from sklearn import metrics
X = np.load('../blobs/X_train.npy')
Y = np.load('../blobs/Y_train.npy')
print 'done loading'
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)
# 2304, 4608, 1152, 576, 10
# 15-20 epochs
clf = DBN([X_train.shape[1], X_train.shape[1]*2, X_train.shape[1]//2, X_train.shape[1]//4, 10], learn_rates=0.01, learn_rate_decays=0.9, momentum=0.9, epochs=10, verbose=1)
clf.fit(X_train, Y_train)
pred = clf.predict(X_test)
score = metrics.f1_score(Y_test, pred)
print("f1-score: %0.3f" % score)
print("classification report:")
print(metrics.classification_report(Y_test, pred))
print("confusion matrix:")
print(metrics.confusion_matrix(Y_test, pred))
mnist = fetch_mldata('mnist-original')
X_train, X_test, y_train, y_test = train_test_split(
(mnist.data / 255.0).astype(np.float32),
mnist.target.astype(np.int32),
test_size=1.0/7.0, random_state=1234)
classifiers = []
if 'dbn' in sys.argv:
from nolearn.dbn import DBN
clf = DBN(
[X_train.shape[1], 300, 10],
learn_rates=0.3,
learn_rate_decays=0.9,
epochs=10,
verbose=1)
classifiers.append(('nolearn.dbn', clf))
if 'sknn' in sys.argv:
from sknn.mlp import Classifier, Layer, Convolution
clf = Classifier(
layers=[
# Convolution("Rectifier", channels=10, pool_shape=(2,2), kernel_shape=(3, 3)),
Layer('Rectifier', units=200),
Layer('Softmax')],
learning_rate=0.02,
learning_rule='momentum',
learning_momentum=0.9,
################
#Neural network#
################
from LoadData import *
from nolearn.dbn import DBN
train_data = load_train_data()
test_data = load_test_data()
print(train_data)
exit(0)
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import ExtraTreesClassifier
clf = DBN([train_data[:, 1:].shape[1], 300, 10],
learn_rates=0.3,
learn_rate_decays=0.9,
epochs=15)
scores = cross_val_score(clf, train_data[:, 1:], train_data[:, 0])
clf.fit(train_data[:, 1:], train_data[:, 0])
y_pred = clf.predict(test_data)
#from sklearn.externals import joblib
#joblib.dump(clf, 'NeuralNetork.pkl')
import csv
with open('submision.csv', 'wb') as csvfile:
spamwriter = csv.writer(csvfile, quotechar=',', quoting=csv.QUOTE_MINIMAL)
spamwriter.writerow(['ImageId'] + ['Label'])
for i in range(len(y_pred)):
spamwriter.writerow([i + 1] + [y_pred[i]])
from sklearn.cross_validation import train_test_split
from sklearn.datasets import fetch_mldata
mnist = fetch_mldata("MNIST original")
X_train, X_test, y_train, y_test = train_test_split(mnist.data / 255.0, mnist.target)
from nolearn.dbn import DBN
clf = DBN([X_train.shape[1], 300, 10], learn_rates=0.3, learn_rate_decays=0.9, epochs=10, verbose=1)
clf.fit(X_train, y_train)
from sklearn.metrics import classification_report
from sklearn.metrics import zero_one_loss
y_pred = clf.predict(X_test)
print "Accuracy:", 1 - zero_one_loss(y_test, y_pred)
print "Classification report:"
print classification_report(y_test, y_pred)
####################################
if __name__=='__main__':
dbn_list=[]
for i in range(2,8):
dat,lab=db_load(tup(i),i)
try:
dbn = joblib.load("pickles/dbn_"+str(tup(i))+"x"+str(i)+".pkl")
dbn_list.append(dbn)
except:
dbn = DBN(
[i*tup(i), 400, 10],
learn_rates = 0.3,
learn_rate_decays = 0.9,
epochs = 50,
verbose = 1
)
dbn.fit(dat,lab)
dbn_list.append(dbn)
joblib.dump(dbn,"pickles/dbn_"+str(tup(i))+"x"+str(i)+".pkl")
finally:
#print dat.shape
#print lab.shape
print dbn_list.__len__()
print ("trained ! ready to predict!")
#print "training report for {}x{}:".format(tup(i),i)
tes,labt=test_load(tup(i),i)
preds=dbn.predict(tes)
sampleClassificationReport=classification_report(labt,preds)
train_x = numpy.load('data/X_train.npy')
test_x = numpy.load('data/X_test.npy')
train_y = numpy.load('data/Y_train.npy')
print train_x.shape
print train_y.shape
print test_x.shape
(train_x,vali_x,train_y,vali_y) = train_test_split(train_x,train_y,test_size = 0.2)
dbn = DBN(
[300,1024,120000],
learn_rates = 0.025,
learn_rate_decays = 0.98,
l2_costs = 0.0001,
minibatch_size=256,
epochs=5,
momentum = 0.9,
#dropouts=0.22,
verbose = 2)
dbn.fit(train_x, train_y)
print 'validation score is:' ,dbn.score(vali_x,vali_y)
result = dbn.predict(test_x)
with open('data/result','w') as f:
for el in result:
f.write(el+'\n')
#predicted_y_proba = dbn.predict_proba(test_x)
<START TRAINING>
- Referred from : http://goo.gl/GBYZvR
"""
# Collect Data
# dataset = unpickle('data_batch_1')
train['data'] = np.asarray(train['data'])
data_train, labels_train = cvt_tastable_set(train)
data_train = data_train.astype('float') / 255.
labels_train = labels_train
# Training Data
n_feat = data_train.shape[1]
n_targets = labels_train.max() + 1
net = DBN(
[n_feat, n_feat / 3, n_targets],
epochs=20,
learn_rates=0.02,
verbose=1
)
net.fit(data_train, labels_train)
# f = file('Deeplearned.save', 'wb')
# cPickle.dump(net, f, protocol=cPickle.HIGHEST_PROTOCOL)
# f.close()
# Test set generation
# image_list = []
# label_index = []
# data_list2 = {'labels': label_index, 'data': image_list}
# test = collect_images(
# dict=data_list2,
# dir_path='./yeragoData/testSet/right_eyes',
# label_addition=True,
def train(args):
print("Loading embeddings.")
fname = "{}/labels.csv".format(args.workDir)
labels = pd.read_csv(fname, header=None).as_matrix()[:, 1]
labels = map(itemgetter(1),
map(os.path.split,
map(os.path.dirname, labels))) # Get the directory.
fname = "{}/reps.csv".format(args.workDir)
embeddings = pd.read_csv(fname, header=None).as_matrix()
le = LabelEncoder().fit(labels)
labelsNum = le.transform(labels)
nClasses = len(le.classes_)
print("Training for {} classes.".format(nClasses))
if args.classifier == 'LinearSvm':
clf = SVC(C=1, kernel='linear', probability=True)
elif args.classifier == 'GridSearchSvm':
print("""
Warning: In our experiences, using a grid search over SVM hyper-parameters only
gives marginally better performance than a linear SVM with C=1 and
is not worth the extra computations of performing a grid search.
""")
param_grid = [
{'C': [1, 10, 100, 1000],
'kernel': ['linear']},
{'C': [1, 10, 100, 1000],
'gamma': [0.001, 0.0001],
'kernel': ['rbf']}
]
clf = GridSearchCV(SVC(C=1, probability=True), param_grid, cv=5)
elif args.classifier == 'GMM': # Doesn't work best
clf = GMM(n_components=nClasses)
# ref:
# http://scikit-learn.org/stable/auto_examples/classification/plot_classifier_comparison.html#example-classification-plot-classifier-comparison-py
elif args.classifier == 'RadialSvm': # Radial Basis Function kernel
# works better with C = 1 and gamma = 2
clf = SVC(C=1, kernel='rbf', probability=True, gamma=2)
elif args.classifier == 'DecisionTree': # Doesn't work best
clf = DecisionTreeClassifier(max_depth=20)
elif args.classifier == 'GaussianNB':
clf = GaussianNB()
# ref: https://jessesw.com/Deep-Learning/
elif args.classifier == 'DBN':
from nolearn.dbn import DBN
clf = DBN([embeddings.shape[1], 500, labelsNum[-1:][0] + 1], # i/p nodes, hidden nodes, o/p nodes
learn_rates=0.3,
# Smaller steps mean a possibly more accurate result, but the
# training will take longer
learn_rate_decays=0.9,
# a factor the initial learning rate will be multiplied by
# after each iteration of the training
epochs=300, # no of iternation
# dropouts = 0.25, # Express the percentage of nodes that
# will be randomly dropped as a decimal.
verbose=1)
if args.ldaDim > 0:
clf_final = clf
clf = Pipeline([('lda', LDA(n_components=args.ldaDim)),
('clf', clf_final)])
clf.fit(embeddings, labelsNum)
fName = "{}/classifier.pkl".format(args.workDir)
print("Saving classifier to '{}'".format(fName))
with open(fName, 'w') as f:
pickle.dump((le, clf), f)
"Label": predictions
}
predictions_table = pd.DataFrame(predictions_dict)
predictions_table.to_csv(csv_path, index=False)
X_train, Y_train = load_training_data()
X_train, Y_train = rotate_dataset(X_train, Y_train)
#X_train, Y_train = nudge_dataset(X_train, Y_train)
n_features = X_train.shape[1]
n_classes = 10
classifier = DBN([n_features, 10, n_classes],
learn_rates=0.01,
learn_rate_decays=0.9,
epochs=1,
verbose=1)
classifier.fit(X_train, Y_train)
test_data = Z
predictions = classifier.predict(test_data)
csv_path = make_predictions_path()
write_predictions_to_csv(predictions)
def __main__(args):
run()
print "Loading data..."
X_train, y_train = load_data("../dataset/%s" % TRAIN_DATA)
# Split data to train and test
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=TEST_SIZE, random_state=0)
X_train = X_train.todense()
X_test = X_test.todense()
# Train --------------------------------------------------------------
print "Training..."
t1 = datetime.now()
dbn = DBN(
[-1, 300, 300, -1],
learn_rates=0.1,
learn_rate_decays=0.9,
epochs=10,
verbose=1)
dbn.fit(X_train, y_train)
print "Training %f secs" % (datetime.now() - t1).total_seconds()
if TEST_SIZE > 0:
tlabel = dbn.predict(X_test)
print 'Error: %f' % error_track_0(tlabel, y_test)
if DUMP:
# Dump model --------------------------------------------------------------
print "Dumping model..."
joblib.dump(dbn, '../model/deep/%s.pkl' % MODEL_NAME)
class DBNRegressor(BaseEstimator, RegressorMixin):
def __init__(
self,
n_hidden_layers=2,
n_units=1000,
epochs=100,
epochs_pretrain=0,
scales=0.05,
real_valued_vis=True,
use_re_lu=False,
uniforms=False,
learn_rates_pretrain=0.1,
learn_rates=0.1,
learn_rate_decays=1.0,
learn_rate_minimums=0.0,
momentum=0.9,
momentum_pretrain=0.9,
l2_costs=0.0001,
l2_costs_pretrain=0.0001,
dropouts=None,
minibatch_size=64,
verbose=2,
fine_tune_callback=None,
nest_compare=True,
nest_compare_pretrain=None,
fan_outs=None,
nesterov=False,
):
self.n_hidden_layers = n_hidden_layers
self.n_units = n_units
self.epochs = epochs
self.epochs_pretrain = epochs_pretrain
self.learn_rates_pretrain = learn_rates_pretrain
self.learn_rates = learn_rates
self.learn_rate_decays = learn_rate_decays
self.learn_rate_minimums = learn_rate_minimums
self.l2_costs_pretrain = l2_costs_pretrain
self.l2_costs = l2_costs
self.momentum = momentum
self.momentum_pretrain = momentum_pretrain
self.verbose = verbose
self.real_valued_vis = real_valued_vis
self.use_re_lu = use_re_lu
self.scales = scales
self.minibatch_size = minibatch_size
if dropouts is None:
dropouts = [0.2] + [0.5] * n_hidden_layers
self.dropouts = dropouts
self.fine_tune_callback = fine_tune_callback
self.nest_compare = nest_compare
self.nest_compare_pretrain = nest_compare_pretrain
self.fan_outs = fan_outs
self.nesterov = nesterov
def fit(self, X, y, X_pretrain=None):
from nolearn.dbn import DBN
if y.ndim == 2:
n_outputs = y.shape[1]
else:
y = y[:, np.newaxis]
n_outputs = 1
params = dict(self.__dict__)
from gdbn.activationFunctions import Linear
params['output_act_funct'] = Linear()
n_units = params.pop('n_units')
n_hidden_layers = params.pop('n_hidden_layers')
if isinstance(n_units, int):
units = [n_units] * n_hidden_layers
else:
units = n_units
units = [X.shape[1]] + units + [n_outputs]
self.dbn = DBN(units, **params)
print X.shape
self.dbn.fit(X, y, X_pretrain=X_pretrain)
def predict(self, X):
return self.dbn.chunked_decision_function(X)
"""
Train the network with 3200 inputs (64x50 values in file)
6 output units (for the different defects)
Lets start with 2 hidden units
"""
# Numbers from example used here
(trainX, testX, trainY, testY) = train_test_split(
dataset / 255.0, dataset.target.astype("int0"), test_size=0.33)
dbn = DBN(
# [[numNodes input layer], numNodes hidden layer, numNodes output layer ]
[trainX.shape[1], 2, 6],
# Learning rate of algorithm
learn_rates=0.3,
# Decay of learn rate
learn_rate_decays=0.9,
# Iterations of training data (epochs)
epochs=10,
# Verbosity level
verbose=1)
dbn.fit(trainX,trainY)
print "trained yo!"
# Evaluate network
#-----------------
preds = dbn.predict(testX)
print classification_report(testY, preds) # Table of accuracies
X_train, X_test, y_train, y_test = cross_validation.train_test_split(train,train_labels,test_size=0.2,random_state=0)
print "Applying a learning algorithm..."
from nolearn.dbn import DBN
X_train=np.array(X_train)
X_test=np.array(X_test)
y_train=np.array(y_train)
y_test=np.array(y_test)
clf = DBN(
[X_train.shape[1], 300, 10],
learn_rates=0.3,
learn_rate_decays=0.9,
epochs=15,
verbose=1,
)
clf.fit(X_train, y_train)
acc_nn = clf.score(X_test,y_test)
print "neural network accuracy: ",acc_nn
y_pred = clf.predict(X_test)
print "Classification report:"
print classification_report(y_test, y_pred)
print("Confusion matrix:\n%s" % metrics.confusion_matrix(y_test,y_pred))
# load the data, scale the data to the range [0, 1] and then construct
# the training and testing splits
dataset = io.loadmat("mnist-original.mat")
data = dataset["data"].T
labels = dataset["label"].flatten().astype("int")
(trainX, testX, trainY, testY) = train_test_split(data / 255.0,
labels,
test_size=0.33)
# train the Deep Belief Network with 784 input units (the flattened,
# 28x28 grayscale image), 300 hidden units, 10 output units (one for
# each possible output classification, which are the digits 1-10)
dbn = DBN([trainX.shape[1], 300, 10],
learn_rates=0.3,
learn_rate_decays=0.9,
epochs=10,
verbose=1)
dbn.fit(trainX, trainY)
# compute the predictions for the test data and show a classification
# report
preds = dbn.predict(testX)
print classification_report(testY, preds)
# randomly select a few of the test instances
for i in np.random.choice(np.arange(0, len(testY)), size=(10, )):
# classify the digit
pred = dbn.predict(np.atleast_2d(testX[i]))
# reshape the feature vector to be a 28x28 pixel image, then change
import numpy as np
from sklearn import cross_validation
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import LinearSVC
from sklearn.linear_model import SGDClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score
from nolearn.dbn import DBN
import timeit
train = pd.read_csv("mnist_train.csv")
features = train.columns[1:]
X = train[features]
y = train['label']
X_train, X_test, y_train, y_test = cross_validation.train_test_split(X/255., y, test_size = 0.1, random_state = 0)
clf_rf = RandomForestClassifier()
clf_rf.fit(X_train, y_train)
y_pred_rf = clf_rf.predict(X_test)
acc_rf = accuracy_score(y_test, y_pred_rf)
"nn"
clf_nn = DBN([X_train.shape[1], 300, 10], learn_rates=0.3, learn_rate_decays=0.9, epochs=50)
clf_nn.fit(X_train, y_train)
acc_nn = clf_nn.score(X_test, y_test)
# this script, this make take a minute -- the 55mb MNIST digit dataset
# will be downloaded)
print "[X] downloading data..."
dataset = datasets.fetch_mldata("MNIST Original")
# scale the data to the range [0, 1] and then construct the training
# and testing splits
(trainX, testX, trainY, testY) = train_test_split(
dataset.data / 255.0, dataset.target.astype("int0"), test_size = 0.33)
# train the Deep Belief Network with 784 input units (the flattened,
# 28x28 grayscale image), 300 hidden units, 10 output units (one for
# each possible output classification, which are the digits 1-10)
dbn = DBN(
[trainX.shape[1], 300, 10],
learn_rates = 0.3,
learn_rate_decays = 0.9,
epochs = 10,
verbose = 1)
dbn.fit(trainX, trainY)
# compute the predictions for the test data and show a classification
# report
preds = dbn.predict(testX)
print classification_report(testY, preds)
# randomly select a few of the test instances
for i in np.random.choice(np.arange(0, len(testY)), size = (10,)):
# classify the digit
pred = dbn.predict(np.atleast_2d(testX[i]))
# reshape the feature vector to be a 28x28 pixel image, then change
orientations=9,
pixels_per_cell=(4, 4),
cells_per_block=(2, 2),
visualise=False)
hogFeatures_test.append(f)
hogFeatures_test_np = np.array(hogFeatures_test)
print "hog features test dimensions", hogFeatures_test_np.shape
print "init len ", len(train_x_main)
for i in range(10):
m = int(len(train_x_main) * (i + 1.0) / 10)
train_x = train_x_main[0:m]
print "len at ", i, " iteration : ", len(train_x)
train_y = train_y_main[0:m]
clf_nn = DBN([train_x.shape[1], 300, 10],
learn_rates=0.3,
learn_rate_decays=0.9,
epochs=15)
clf_nn.fit(train_x, train_y)
acc_nn = clf_nn.score(test_x, test_y)
arr_acc.append(acc_nn)
hogFeatures_train = []
for feature in train_x:
f = hog(feature.reshape((28, 28)),
orientations=9,
pixels_per_cell=(4, 4),
cells_per_block=(2, 2),
visualise=False)
hogFeatures_train.append(f)
hogFeatures_train_np = np.array(hogFeatures_train)
print "hog features train dimensions", hogFeatures_train_np.shape
print "length of hog train np ", len(hogFeatures_train)
