def closed_form_ridge_regression(self, training_data, training_results):
cv = crossvalidate.crossvalidate()
l = cv.determine_lambda(training_data, training_results)
weights = self.closed_form_lin_regression(training_data,
training_results, l)
return weights
def gradient_descent_ridge_regression(self, training_data,
training_results):
cv = crossvalidate.crossvalidate()
l = cv.determine_lambda(training_data, training_results)
weights = self.gradient_descent_lin_regression(training_data,
training_results, l)
return weights
P = 1
svmclassify = trainsvm(xTr, yTr, C, 'rbf', P)
# Get training error of initial classifier
train_preds = svmclassify(xTr)
train_error = np.mean(train_preds != yTr)
print("Train error:", train_error)
# Visualize svmclassify on the training set
visdecision(xTr, yTr, svmclassify)
plt.show()
# Do crossvalidation (you will want to test different parameters)
Cs = (2.0**np.linspace(-1, 6, 200)) #linspace (start, stop, num=50)
Ps = (np.linspace(0.01, 5, 200))
bestC, bestP, lowest_error, errors = crossvalidate(xTr, yTr, 'rbf', Cs, Ps)
print("Best C:", bestC)
print("Best P:", bestP)
#print(lowest_error)
# Save the best parameters to be run on test data by the autorader
best_parameters = {'C': bestC, 'P': bestP}
pickle.dump(best_parameters, open('best_parameters.pickle', 'wb'))
# Don't forget to commit best_parameters.pickle!
# Plot the performance of different parameters (this looks better with more parameter choices)
# x,y = np.array(np.meshgrid(Cs, Ps))
# plt.contourf(x, y, errors)
# plt.colorbar()
# plt.xlabel('C')
def _runExperiment(train, test, cross_epochs, epochs, batch_size, classifierName):
'''
Runs the experiment returning the classifier generated
'''
print 'Running the experiment'
print ' training data: ', train
print ' testing data: ', test
print ' epochs: ', epochs
print ' batch size: ', batch_size
print ' classifier: ', classifierName
print
print 'Loading data ... ',
sys.stdout.flush()
start = time.clock()
with open(train, 'r') as trainfile:
xdata, ydata = pickle.load(trainfile)
xdata = preprocessData(xdata, reshape=(classifierName != 'ConvNet'))
# Split into verify set and data set
xlen = len(xdata)
xverify = xdata[:xlen/5]
yverify = ydata[:xlen/5]
xdata = xdata[xlen/5:]
ydata = ydata[xlen/5:]
print 'done'
kb_used = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
print ' elapsed time: ', time.clock() - start
print ' memory used: ', kb_used, 'KB,', float(kb_used) / 2**10, 'MB'
print ' xshape: ', xdata.shape
print ' yshape: ', ydata.shape
print
print 'Perform cross-validation'
k = 5
rValues = [0.0001, 0.001, 0.01, 0.1, 0.5]
CValues = [0.0001, 0.001, 0.01, 0.1]
hiddenDims = [1, 20, 40, 80]
out_im1_amounts = [10, 20]
out_im2_amounts = [20, 50]
rValues = [0.05]
CValues = [0.01]
hiddenDims = [20]
out_im1_amounts = [10]
out_im2_amounts = [20]
mlpLearner = lambda dim_in, dim_hidden, r, C: \
Mlp(dim_in, dim_hidden, 2, r, C)
im_shape = (40, 68)
convnetLearner = lambda dim_in, out_im1, out_im2, dim_hidden, r, C: \
ConvNet(
im_shape,
(out_im1, out_im2),
((5, 5), (5, 5)),
((2, 2), (2, 2)),
batch_size,
dim_hidden,
2,
r,
C,
)
classifierMap = {
'MLP': (
mlpLearner,
list(itertools.product(hiddenDims, rValues, CValues)),
['hidden-dims', 'r', 'C']
),
'SVM': (SVM, list(itertools.product(rValues, CValues)), ['r', 'C']),
'Perceptron': (Perceptron, [(x,) for x in rValues], ['r']),
'AveragedPerceptron': (
AveragedPerceptron,
[(x,) for x in rValues],
['r']
),
'LogisticRegression': (
LogisticRegression,
list(itertools.product([2], rValues, CValues)),
['dim_out', 'r', 'C']
),
'ConvNet': (
convnetLearner,
list(itertools.product(out_im1_amounts, out_im2_amounts, hiddenDims, rValues, CValues)),
['kernel_1', 'kernel_2', 'hidden-dims', 'r', 'C']
),
}
algorithm, hyperparams, hypernames = classifierMap[classifierName]
if classifierName in ('LogisticRegression', 'MLP'):
ydata -= ydata.min()
ydata /= ydata.max()
yverify -= yverify.min()
yverify /= yverify.max()
start = time.clock()
print 'Doing {k}-cross validation sequentially'.format(k=k)
print ' params: ', hypernames
for i in xrange(len(hypernames)):
print ' {0} values: '.format(hypernames[i]), sorted(set([x[i] for x in hyperparams]))
params = crossvalidate(algorithm, xdata, ydata, k, cross_epochs, batch_size,
hyperparams, hypernames)
print ' elapsed time: ', time.clock() - start
print ' best params: ', params
# I tried to get threading to work, but it just didn't do very well.
# It took a lot longer than single threaded, maybe because theano can't be
# parallelized like that...
#print 'Doing {k}-cross validation multithreaded'.format(k=k)
#params = crossvalidate_threaded(AveragedPerceptron, xdata, ydata, k,
# epochs, batch_size,
# hyperparams, hypernames)
kb_used = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
print ' memory used before gc:', kb_used, 'KB,', float(kb_used) / 2**10, 'MB'
gc.collect()
kb_used = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
print ' memory used after gc: ', kb_used, 'KB,', float(kb_used) / 2**10, 'MB'
print
print 'Training Perceptron'
print ' epochs: ', epochs
print ' batch size: ', batch_size
print ' training ',
sys.stdout.flush()
start = time.clock()
classifier = algorithm(xdata.shape[1], *params)
classifier.train(xdata, ydata, epochs, batch_size, xverify=xverify, yverify=yverify)
print ' done'
predictions = classifier.predict(xdata)
print ' elapsed time: ', time.clock() - start
print ' training accuracy: ', np.sum(ydata == predictions) / float(len(ydata))
kb_used = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
print ' memory used before gc:', kb_used, 'KB,', float(kb_used) / 2**10, 'MB'
del xdata
del ydata
gc.collect()
kb_used = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
print ' memory used after gc: ', kb_used, 'KB,', float(kb_used) / 2**10, 'MB'
print
print 'Loading test data ... ',
sys.stdout.flush()
start = time.clock()
with open(test, 'r') as testfile:
testx, testy = pickle.load(testfile)
testx = preprocessData(testx, reshape=(classifierName != 'ConvNet'))
if classifierName in ('LogisticRegression', 'MLP'):
testy -= testy.min()
testy /= testy.max()
print ' done'
kb_used = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
print ' elapsed time: ', time.clock() - start
print ' memory used: ', kb_used, 'KB,', float(kb_used) / 2**10, 'MB'
print ' xshape: ', testx.shape
print ' yshape: ', testy.shape
print
print 'Running classifier on test set'
start = time.clock()
predictions = classifier.predict(testx)
print ' elapsed time: ', time.clock() - start
print ' test accuracy: ', np.sum(testy == predictions) / float(len(testy))
print
return classifier
def testLearner(name, trainExamples, testExamples, crossepochs=10,
epochs=10, batchSize=1, learnerName='SVM'):
'''
Trains an averaged Perceptron classifier from the training examples and
then calculates the accuracy of the generated classifier on the test
examples.
Prints out the results to the console
'''
featuresList = np.asarray([x.features for x in trainExamples], dtype=theano.config.floatX)
labels = np.asarray([x.label for x in trainExamples], dtype=theano.config.floatX)
testFeatures = np.asarray([x.features for x in testExamples], dtype=theano.config.floatX)
testLabels = np.asarray([x.label for x in testExamples], dtype=theano.config.floatX)
rvalues = [0.01, 0.05, 0.1, 0.5]
Cvalues = [0.001, 0.005, 0.01, 0.05]
dimvalues = [10, 20]
logreglearner = lambda dim_in, r, C: LogisticRegression(dim_in, 2, r, C)
mlplearner = lambda dim_in, dim_hidden, r, C: Mlp(dim_in, dim_hidden, 2, r, C)
learnerMap = {
'Perceptron': (Perceptron, [(x,) for x in rvalues], ['r']),
'AveragedPerceptron': (AveragedPerceptron, [(x,) for x in rvalues], ['r']),
'SVM': (SVM, list(itertools.product(rvalues, Cvalues)), ['r', 'C']),
'LogisticRegression': (
logreglearner,
list(itertools.product(rvalues, Cvalues)),
['r', 'C']
),
'MLP': (
mlplearner,
list(itertools.product(dimvalues, rvalues, Cvalues)),
['hidden-dimension', 'r', 'C']
),
}
learner, hypers, names = learnerMap[learnerName]
logisticLearners = ('LogisticRegression', 'MLP')
if learnerName in logisticLearners:
labels -= labels.min()
labels /= labels.max()
testLabels -= testLabels.min()
testLabels /= testLabels.max()
k = 5
print 'Performing cross-validation'
print ' dataset: ', name
print ' learner: ', learnerName
print ' k: ', k
print ' parameters: ', names
bestHyper = crossvalidate(learner, featuresList, labels, k, crossepochs,
batchSize, hypers, names)
print ' best params: ', names, '=', bestHyper
print
p = learner(len(featuresList[0]), *bestHyper)
print 'training ' + name + ' ',
sys.stdout.flush()
p.train(featuresList, labels, epochs, batchSize)
print ' done'
# Test accuracy on the training set
predictions = p.predict(featuresList)
accuracy = np.sum(labels == predictions) / float(len(labels))
print name, ' train accuracy: ', accuracy
# Test accuracy on the testing set
predictions = p.predict(testFeatures)
accuracy = np.sum(testLabels == predictions) / float(len(testLabels))
print name, ' test accuracy: ', accuracy
