def number_layer_exp(X, y):
# split to training and validation set
train_x, train_y, valid_x, valid_y = split(X, y, 0.7)
histories = list()
for n_hidden_layers in range(1, 11, 2):
mlp_clf = mlp.MLPClassifier(hidden_layer_sizes=128,
n_hidden_layers=n_hidden_layers,
activation='relu',
solver='sgd',
momentum=0.,
nesterov=False,
loss='cross_entropy',
weight_initializer='orthogonal',
bias_initializer='zeros',
batch_size=32,
learning_rate=0.01,
lr_decay_on_plateau=0.5,
lr_decay_patience=15,
early_stop_patience=40)
history = mlp_clf.fit(train_x,
train_y,
valid_set=(valid_x, valid_y),
epochs=2000,
verbose=True)
print('n_hidden_layer:', n_hidden_layers)
print('loss:', history['loss'][-1])
print('acc:', history['acc'][-1])
print('valid_loss:', history['valid_loss'][-1])
print('valid_acc:', history['valid_acc'][-1])
print('n_epoch:', len(history['loss']))
histories.append(history)
pickle.dump(histories, open('n_hidden_layer_history.pl', 'wb'))
def q4():
trainingData, validationData, testData, trainingLabels, validationLabels, testLabels = getNumpyData(
5000, 1000)
full_size = 5000
percents = np.linspace(.1, 1, 10)
print(percents)
classes = range(10)
data = {'svm': [], 'numceptron': [], 'mlp': []}
for i in percents:
size = int(i * full_size)
X = trainingData[0:size]
y = trainingLabels[0:size]
models = [
svm.SVMClassifier(classes),
numceptron.numceptronClassifier(classes, 20),
mlp.MLPClassifier(classes, 1000, 200)
]
for j, m in enumerate(models):
m.train(X, y, validationData, validationLabels)
pred = m.classify(testData)
acc = 1.0 * np.sum(pred == testLabels) / len(testLabels)
data[m.type].append(acc)
print('{}: {} acc: {}'.format(j, m.type, acc))
for key in data.keys():
plt.figure()
plt.plot(percents, data[key])
plt.ylabel('Accuracy on test set')
plt.xlabel('Percent of training set used')
plt.title('{} performance on test set'.format(key))
plt.savefig('data/{}_test.png'.format(key))
print(data)
data = dataRaw.data[:, 0:-1]
labels = dataRaw.data[:, -1].reshape(-1, 1)
## Define the initial Parameters ##
LR = 0.1
DET = 10
SHUFFLE = False
MOMENTUM = 0.5
VALIDATION_SIZE = 0.0
HOTENCODING = True
# data = [[0,0],[0,1]]
# BClass = mlp.MLPClassifier(lr = LR,momentum = MOMENTUM, shuffle = SHUFFLE, deterministic = DET, hidden_layer_widths =[2,2,2] ,validationSize = VALIDATION_SIZE, allWeightsValue = 0)
BClass = mlp.MLPClassifier(lr=LR,
momentum=MOMENTUM,
shuffle=SHUFFLE,
deterministic=DET,
validationSize=VALIDATION_SIZE,
allWeightsValue=0,
isHotEncoding=HOTENCODING)
BClass.fit(data, labels)
scores = BClass.score(data, labels)
print("Score ", scores)
# print("Weights")
# print(BClass.get_weights())
list2csv.write_to_csv(BClass.get_weights(), "weightsP1D1.csv")
# clf = MLPClassifier(hidden_layer_sizes=(4,), activation = 'logistic',solver = 'sgd',alpha = MOMENTUM,learning_rate_init = LR, max_iter=2,shuffle = SHUFFLE).fit(data, labels)
print("DATA SET 2")
arff_path = r"training/data_banknote_authentication.arff"
dataRaw = arff.Arff(arff=arff_path, label_count=1)
def readCommand(argv):
"Processes the command used to run from the command line."
from optparse import OptionParser
parser = OptionParser(USAGE_STRING)
parser.add_option('-c',
'--classifier',
help=default('The type of classifier'),
choices=['mostFrequent', 'perceptron', 'mlp', 'svm'],
default='mostFrequent')
parser.add_option('-t',
'--training',
help=default('The size of the training set'),
default=TRAINING_SET_SIZE,
type="int")
parser.add_option('-w',
'--weights',
help=default('Whether to print weights'),
default=False,
action="store_true")
parser.add_option('-i',
'--iterations',
help=default("Maximum iterations to run training"),
default=3,
type="int")
parser.add_option('-s',
'--test',
help=default("Amount of test data to use"),
default=TEST_SET_SIZE,
type="int")
options, otherjunk = parser.parse_args(argv)
if len(otherjunk) != 0:
raise Exception('Command line input not understood: ' + str(otherjunk))
args = {}
# Set up variables according to the command line input.
print "Doing classification"
print "--------------------"
print "classifier:\t\t" + options.classifier
print "training set size:\t" + str(options.training)
printImage = ImagePrinter(DIGIT_DATUM_WIDTH, DIGIT_DATUM_HEIGHT).printImage
featureFunction = basicFeatureExtractorDigit
legalLabels = range(10)
if options.training <= 0:
print "Training set size should be a positive integer (you provided: %d)" % options.training
print USAGE_STRING
sys.exit(2)
if (options.classifier == "mostFrequent"):
classifier = mostFrequent.MostFrequentClassifier(legalLabels)
elif (options.classifier == "mlp"):
classifier = mlp.MLPClassifier(legalLabels, options.iterations)
elif (options.classifier == "perceptron"):
classifier = perceptron.PerceptronClassifier(legalLabels,
options.iterations)
elif (options.classifier == "svm"):
classifier = svm.SVMClassifier(legalLabels)
else:
print "Unknown classifier:", options.classifier
print USAGE_STRING
sys.exit(2)
args['classifier'] = classifier
args['featureFunction'] = featureFunction
args['printImage'] = printImage
return args, options
def activation_function_exp(X, y):
# split to training and validation set
train_x, train_y, valid_x, valid_y = split(X, y, 0.7)
relu_clf = mlp.MLPClassifier(hidden_layer_sizes=128,
n_hidden_layers=9,
activation='relu',
solver='sgd',
momentum=0.,
nesterov=False,
loss='cross_entropy',
weight_initializer='orthogonal',
bias_initializer='zeros',
batch_size=32,
learning_rate=0.01,
lr_decay_on_plateau=0.5,
lr_decay_patience=15,
early_stop_patience=40)
leaky_relu_clf = mlp.MLPClassifier(hidden_layer_sizes=128,
n_hidden_layers=9,
activation='leaky_relu',
solver='sgd',
momentum=0.,
nesterov=False,
loss='cross_entropy',
weight_initializer='orthogonal',
bias_initializer='zeros',
batch_size=32,
learning_rate=0.01,
lr_decay_on_plateau=0.5,
lr_decay_patience=15,
early_stop_patience=40)
selu_clf = mlp.MLPClassifier(hidden_layer_sizes=128,
n_hidden_layers=9,
activation='selu',
solver='sgd',
momentum=0.,
nesterov=False,
loss='cross_entropy',
weight_initializer='orthogonal',
bias_initializer='zeros',
batch_size=32,
learning_rate=0.01,
lr_decay_on_plateau=0.5,
lr_decay_patience=15,
early_stop_patience=40)
print('ReLU Start!')
relu_history = relu_clf.fit(train_x,
train_y,
valid_set=(valid_x, valid_y),
epochs=2000,
verbose=True)
print('Leaky Start!')
leaky_relu_history = leaky_relu_clf.fit(train_x,
train_y,
valid_set=(valid_x, valid_y),
epochs=2000,
verbose=True)
print('SELU Start!')
selu_history = selu_clf.fit(train_x,
train_y,
valid_set=(valid_x, valid_y),
epochs=2000,
verbose=True)
pickle.dump(relu_history, open('relu_history.pl', 'wb'))
pickle.dump(leaky_relu_history, open('leaky_relu_history.pl', 'wb'))
pickle.dump(selu_history, open('selu_history.pl', 'wb'))
def momentum_exp(X, y):
train_x, train_y, valid_x, valid_y = split(X, y, 0.7)
no_momentum_clf = mlp.MLPClassifier(hidden_layer_sizes=128,
n_hidden_layers=9,
activation='relu',
solver='sgd',
momentum=0.,
nesterov=False,
loss='cross_entropy',
weight_initializer='orthogonal',
bias_initializer='zeros',
batch_size=32,
learning_rate=0.01,
lr_decay_on_plateau=0.5,
lr_decay_patience=15,
early_stop_patience=40)
momentum_clf = mlp.MLPClassifier(hidden_layer_sizes=128,
n_hidden_layers=9,
activation='relu',
solver='sgd',
momentum=0.4,
nesterov=False,
loss='cross_entropy',
weight_initializer='orthogonal',
bias_initializer='zeros',
batch_size=32,
learning_rate=0.01,
lr_decay_on_plateau=0.5,
lr_decay_patience=20,
early_stop_patience=60)
nesterov_clf = mlp.MLPClassifier(hidden_layer_sizes=128,
n_hidden_layers=9,
activation='relu',
solver='sgd',
momentum=0.9,
nesterov=True,
loss='cross_entropy',
weight_initializer='orthogonal',
bias_initializer='zeros',
batch_size=32,
learning_rate=0.01,
lr_decay_on_plateau=0.5,
lr_decay_patience=20,
early_stop_patience=80)
print('no momentum start!')
# no_momentum_history = no_momentum_clf.fit(train_x, train_y, valid_set=(valid_x, valid_y), epochs=2000, verbose=True)
print('momentum start!')
momentum_history = momentum_clf.fit(train_x,
train_y,
valid_set=(valid_x, valid_y),
epochs=2000,
verbose=True)
print('nesterov start!')
nesterov_history = nesterov_clf.fit(train_x,
train_y,
valid_set=(valid_x, valid_y),
epochs=2000,
verbose=True)
# pickle.dump(no_momentum_history, open('no_momentum_history.pl', 'wb'))
pickle.dump(momentum_history, open('momentum_history.pl', 'wb'))
pickle.dump(nesterov_history, open('nesterov_history.pl', 'wb'))
return X, y
if __name__ == '__main__':
X, y = get_data(filename='cross200')
print(X.shape, y.shape)
train_x, train_y, valid_x, valid_y = split(X, y, 0.7)
mlp_clf = mlp.MLPClassifier(hidden_layer_sizes=32,
n_hidden_layers=8,
activation='selu',
solver='sgd',
momentum=0.4,
nesterov=False,
loss='cross_entropy',
weight_initializer='orthogonal',
bias_initializer='zeros',
batch_size=16,
learning_rate=0.01,
lr_decay_on_plateau=0.5,
lr_decay_patience=40,
early_stop_patience=100)
mlp_clf.fit(train_x,
train_y,
valid_set=(valid_x, valid_y),
epochs=2000,
verbose=True)
predict_y = mlp_clf.predict(X).argmax(axis=1)
pickle.dump(X, open('cross200_x.pl', 'wb'))
pickle.dump(predict_y, open('cross200_predict.pl', 'wb'))