def __init__(self, X: csr_matrix, Y: np.array, tune_parameters=False):
super().__init__(X, Y, tune_parameters=False)
input_layer, output_layer = self.X.shape[1], len(np.unique(Y))
inp = tn.layers.base.Input(size=input_layer, sparse='csr')
self.clf = tn.Classifier(layers=[
inp, (100, 'linear'), (50, 'norm:mean+relu'), output_layer
])
def build(self, *hiddens):
input = dict(form='input',
size=self.NUM_INPUTS,
sparse='csr',
name='in')
return theanets.Classifier([input] + list(hiddens) +
[self.NUM_CLASSES])
def rcnn_train(imdb):
use_gpu = False
opts = Opts()
opts.net_def_file = './model-defs/rcnn_batch_256_output_fc7.prototxt'
conf = [imdb.name, 1, 'cachedir', 'cachedir/' + imdb.name]
rcnn_model = RcnnModel(opts.net_def_file, opts.net_file, opts.cache_name)
# rcnn_load_model(rcnn_model, use_gpu)
rcnn_model.detectors.crop_mode = opts.crop_mode
rcnn_model.detectors.crop_padding = opts.crop_padding
rcnn_model.classes = imdb.classes
rcnn_model.opts = opts
X, y = rcnn_get_all_feature(imdb, rcnn_model)
np.savez('feat', X=X, y=y)
classifier = LogisticRegression(class_weight='balanced',
solver='lbfgs',
multi_class='multinomial',
verbose=1,
n_jobs=-1,
max_iter=1000)
classifier.fit(X, y)
# climate.enable_default_logging()
net = theanets.Classifier(layers=[4096, 21])
net.train((X, y),
algo='sgd',
learning_rate=0.1,
momentum=0.9,
save_every=60.0,
save_progress='net.{}.netsave',
validate_every=100)
def _build_ann(self):
hidden_neurons = 2 * (len(self.inputs) + len(self.languages)) // 3
self._net = theanets.Classifier([
len(self.inputs), {
'size': hidden_neurons,
'activation': 'tanh'
},
len(self.languages)
])
def main(input_file, model_path):
batch_size = 128
nb_classes = 62 # A-Z, a-z and 0-9
nb_epoch = 2
# Input image dimensions
img_rows, img_cols = 32, 32
# Path of data files
path = input_file
# Load the preprocessed data and labels
X_train_all = np.load(path + "/trainPreproc_" + str(img_rows) + "_" +
str(img_cols) + ".npy")
Y_train_all = np.load(path + "/labelsPreproc.npy")
X_train, X_val, Y_train, Y_val = \
train_test_split(X_train_all, Y_train_all, test_size=0.25, stratify=np.argmax(Y_train_all, axis=1))
print X_train.shape
labels = convert_(Y_train)
validation = convert_(Y_val)
X_train = X_train.reshape(
(X_train.shape[0], X_train.shape[2] * X_train.shape[3]))
X_val = X_val.reshape((X_val.shape[0], X_val.shape[2] * X_val.shape[3]))
print 'Training...'
class_input = 62
climate.enable_default_logging()
# Build a classifier model with 100 inputs and 10 outputs.
net = theanets.Classifier(layers=[X_train.shape[1], class_input])
X_train = X_train.astype('f')
labels = labels.astype('i')
X_val = X_val.astype('f')
validation = validation.astype('i')
train = X_train, labels
valid = X_val, validation
arg = 'adadelta'
net.train(train,
valid,
algo=arg,
learning_rate=1e-10,
momentum=0.00001,
input_noise=0.3,
hidden_l1=0.1)
print 'saving model paramters to {}'.format(model_path)
with open(model_path, 'wb') as fid:
pickle.dump(net, fid)
print 'Done.'
def get_model(train_data, train_labels, validation_data, validation_labels, model_fname, output_size, hidden_layer_size):
minimal_accuracy = 0.95
# 1. create a model -- here, a regression model.
print "fname", model_fname
if os.path.exists(model_fname + "..."):
print "Loading an existing model..."
net = theanets.Classifier.load(model_fname)
net._rng = 13
valid_acc = np.sum(net.predict(validation_data) == np.int32(validation_labels)) / float(len(validation_labels))
train_acc = np.sum(net.predict(train_data) == np.int32(train_labels)) / float(len(train_labels))
print "train_acc, valid_acc", train_acc, valid_acc
if valid_acc > minimal_accuracy:
return net
else:
input_size = train_data.shape[1]
print "Creating a new model..."
if hidden_layer_size > 0:
hidden_layer = dict(name='hidden1', size=hidden_layer_size, std=1. / hidden_layer_size ** 0.5)
layers = [theanets.layers.base.Input(size=input_size, sparse='csr'), hidden_layer, dict(size=output_size, diagonal=1)]
else:
layers = [theanets.layers.base.Input(size=input_size, sparse='csr'), output_size]
net = theanets.Classifier(layers, loss='xe')
# 2. train the model.
print "Training..."
alpha = 1.
for eta in [0.01, 0.05]:
print '(eta, alpha)', (eta, alpha)
count = 0
for train, valid in net.itertrain([train_data, np.int32(train_labels)],
valid=[validation_data, np.int32(validation_labels)],
algo='sgd',
learning_rate=eta,
hidden_l1=alpha):
net.save(model_fname)
valid_acc = np.sum(net.predict(validation_data) == np.int32(validation_labels)) / float(len(validation_labels))
print 'valid acc', valid_acc
count += 1
if valid_acc > minimal_accuracy or count == 2:
break
return net
percent should be an int between 1 and 99
'''
s = int(percent * len(data) / 100)
tdata = data[0:s]
vdata = data[s:]
tlabels = labels[0:s]
vlabels = labels[s:]
return ((tdata, tlabels), (vdata, vlabels))
# make a validation set from the train set
train, valid = split_validation(90, data, labels)
# build our classifier
print "We're building a RBM of 1 input layer node, 4 hidden layer nodes, and an output layer of 4 nodes. The output layer has 4 nodes because we have 4 classes that the neural network will output."
cnet = theanets.Classifier([1, 4, 4])
cnet.train(train, valid, algo='layerwise', patience=1, max_updates=mupdates)
cnet.train(train, valid, algo='rprop', patience=1, max_updates=mupdates)
print "%s / %s " % (sum(cnet.classify(data) == labels), tsize)
print "%s / %s " % (sum(cnet.classify(test_data) == test_labels), tsize)
# so what does that output layer look like?
print "The output layer looks something like:"
print "For %s we get %s which is interpreted as class: %s -- but it was %s" % (
data[0:1], cnet.predict_proba(data[0:1]), cnet.classify(
data[0:1]), labels[0])
# now that's kind of interesting, an accuracy of .3 to .5 max
# still pretty inaccurate, but 1 sample might never be enough.
def _build(self, *hiddens):
return theanets.Classifier(
[self.NUM_INPUTS] + list(hiddens) + [self.NUM_CLASSES])
def _build(self, *hiddens):
return theanets.Classifier([self.NUM_INPUTS] + list(hiddens) +
[self.NUM_CLASSES],
sparse_input='csr')
weights = np.ones_like(labels)
weights[labels.nonzero()] *= 10
def split(a, b):
return [
samples[a:b].astype('float32'), labels[a:b].astype('int32'),
weights[a:b].astype('float32')
]
train = split(0, 9000)
valid = split(9000, 10000)
net = theanets.Classifier(
layers=(100, 10, 2),
weighted=True,
)
net.train(train, valid)
truth = valid[1]
print('# of true 1s:', truth.sum())
guess = net.predict(valid[0])
print('# of predicted 1s:', guess.sum())
cm = sklearn.metrics.confusion_matrix(truth, guess)
print('confusion matrix (true class = rows, predicted class = cols):')
print(cm)
def main(data_file, test_file):
try:
# Load the data.
_data = np.loadtxt(data_file,
delimiter=',',
usecols=range(3),
dtype=DTYPE)
_test = np.loadtxt(test_file,
delimiter=',',
usecols=range(3),
dtype=DTYPE)
except Exception:
print "Could not load the data in " + data_file
return 1
# Separate the training dataset from it's labels.
data = np.array([(_data[i][0], _data[i][1])
for i in xrange(_data.shape[0])])
labels_t = np.array([_data[i][2] for i in xrange(_data.shape[0])])
labels_n = np.array([
CLASS2NUM(labels_t[i]) for i in xrange(labels_t.shape[0])
]).astype(np.int32)
# Separate the test dataset from it's labels. The labels are stored for checking later.
test = np.array([(_test[i][0], _test[i][1])
for i in xrange(_test.shape[0])])
test_real = np.array([_test[i][2] for i in xrange(_test.shape[0])])
# Plot the data if needed and able to.
if PLOT and plt is not None:
if XKCD:
with plt.xkcd():
plot_dataset(data, labels_t)
plot_test_input(test, test_real)
else:
plot_dataset(data, labels_t)
plot_test_input(test, test_real)
# Create the classifiers.
cnet_1 = tn.Classifier([2, 4, 3])
cnet_2 = tn.Classifier([2, 8, 4, 3])
cnet_3 = tn.Classifier([2, 8, 6, 4, 3])
# Shuffle and split the training data and labels.
tu.joint_shuffle(data, labels_n)
train, valid = tu.split_validation(90, data, labels_n)
print "*******************************************************************"
print "* TESTING CLASSIFIER: 2, 4, 3 *"
print "*******************************************************************"
test_net(cnet_1, train, valid, test, test_real, "[2,4,3]")
print "*******************************************************************"
print "* TESTING CLASSIFIER: 2, 8, 4, 3 *"
print "*******************************************************************"
test_net(cnet_2, train, valid, test, test_real, "[2,8,4,3]")
print "*******************************************************************"
print "* TESTING CLASSIFIER: 2, 8, 6, 4, 3 *"
print "*******************************************************************"
test_net(cnet_3, train, valid, test, test_real, "[2,8,6,4,3]")
return 0
def _build(self, *hiddens, **kwargs):
return theanets.Classifier(
layers=(784, ) + hiddens + (10, ),
activation='logistic',
**kwargs)
def _build(self, *hiddens):
return theanets.Classifier((self.DIGIT_SIZE, ) + hiddens + (10, ))
def g(x, y, z):
return round(x) or round(y) or round(z)
def npg(vec):
return g(vec[0], vec[1], vec[2])
possible_points = rnd.uniform(0.0, 0.2, (100, 3))
unlikely_points = rnd.uniform(0.0, 1, (100, 3))
points = np.concatenate((possible_points, unlikely_points))
labels = np.apply_along_axis(npf, 1, points)
glabels = np.apply_along_axis(npg, 1, points)
points = points.astype(np.float32)
labels = labels.astype(np.float32)
ilabels = labels.astype(np.int32)
iglabels = glabels.astype(np.int32)
#res = list()
#for i in range(2,18):
cnet = theanets.Classifier([3, 3, 3, ('softmax', 2)])
cnet.train([points, ilabels], algo='layerwise', patience=1)
cnet.train([points, ilabels], algo='rprop', patience=1)
cgnet = theanets.Classifier([3, ('tanh', 3), ('softmax', 2)])
cgnet.train([points, iglabels], algo='rprop', patience=1)
rnet = theanets.Regressor([3, 3, 1])
rnet.train([points, labels], algo='rprop', patience=10, batch_size=4)
100 * tm['acc']))
import climate
import theanets
from sklearn.datasets import make_classification
from sklearn.metrics import confusion_matrix
climate.enable_default_logging()
# Create a classification dataset.
X, y = make_classification(n_samples=3000,
n_features=100,
n_classes=10,
n_informative=10)
X = X.astype('f')
y = y.astype('i')
cut = int(len(X) * 0.8) # training / validation split
train = X[:cut], y[:cut]
valid = X[cut:], y[cut:]
# Build a classifier model with 100 inputs and 10 outputs.
net = theanets.Classifier([100, 10])
# Train the model using SGD with momentum.
net.train(train, valid, algo='sgd', learning_rate=1e-4, momentum=0.9)
# Show confusion matrices on the training/validation splits.
for label, (X, y) in (('training:', train), ('validation:', valid)):
print(label)
print(confusion_matrix(y, net.predict(X)))
#!/usr/bin/env python
import matplotlib.pyplot as plt
import theanets
from utils import load_mnist, plot_layers
train, valid, _ = load_mnist(labels=True)
N = 10
net = theanets.Classifier([784, N * N, ('softmax', 10)])
net.train(train, valid, min_improvement=0.001)
plot_layers([net.find('hid1', 'w'), net.find('out', 'w')])
plt.tight_layout()
plt.show()
#!/usr/bin/env python
import matplotlib.pyplot as plt
import theanets
from utils import load_mnist, plot_layers, plot_images
net = theanets.Classifier(
layers=(784, 1024, 256, 64, ('softmax', 10)),
)
# first, run an unsupervised layerwise pretrainer.
train, valid, _ = load_mnist()
net.train(train, valid,
algo='pretrain',
patience=1,
min_improvement=0.1,
train_batches=100)
# second, run a supervised trainer on the classifier model.
train, valid, _ = load_mnist(labels=True)
net.train(train, valid, min_improvement=0.01, train_batches=100)
plot_layers([net.find(i, 'w') for i in (1, 2, 3)])
plt.tight_layout()
plt.show()
def create_algorithm(train, valid, config, n_features, n_targets, plots=False):
'''Configure and train a theanets neural network
Args
----
train: tuple (ndarray, ndarray)
Tuple containing feature and target values for training
valid: tuple (ndarray, ndarray)
Tuple containing feature and target values for training validation
test: tuple (ndarray, ndarray)
Tuple containing feature and target values for testing
config: dict
Dictionary of network configuration parameters
n_features: int
Number of features (inputs) for configuring input layer of network
n_targets: int
Number of targets (outputs) for configuring output layer of network
plots: bool
Switch for generating diagnostic plots after each network training
Returns
-------
net: theanets.Classifier object
Neural network object
accuracy: float
Accuracy value of the network configuration from the validation dataset
monitors: dict
Dictionary of "monitor" objects produced during network training
Contains two labels 'train' and 'valid' with the following attributes:
- 'loss': percentage from loss function (default: cross-entropy)
- 'err': percentage of error (default: )
- 'acc': percentage of accuracy (defualt: true classifications)
'''
from collections import OrderedDict
import theanets
def plot_monitors(attrs, monitors_train, monitors_valid):
import matplotlib.pyplot as plt
import seaborn
seaborn.set_style('whitegrid')
labels = {'loss': 'Loss', 'err': 'Error', 'acc': 'Accuracy'}
fig, axes = plt.subplots(1, len(attrs), sharex=True)
legend_on = True
for ax, attr in zip(axes, attrs):
ax.yaxis.label.set_text(labels[attr])
ax.xaxis.label.set_text('Epic')
ax.plot(monitors['train'][attr], label='Training')
ax.plot(monitors['valid'][attr], label='Validation')
if legend_on:
ax.legend(loc='upper right')
legend_on = False
plt.show()
# Keep seaborn from messing up confusion matrix plots
seaborn.reset_orig()
return None
# Build neural net with defined configuration
hidden_layers = [
config['hidden_nodes'],
] * config['hidden_layers']
net = theanets.Classifier([
n_features,
] + hidden_layers + [
n_targets,
])
# SGD converges to minima/maxima faster with momentum
# NAG, ADADELTA, RMSProp have equivalents of parameter specific momentum
if config['algorithm'] is 'sgd':
config['momentum'] = 0.9
# Create dictionary for storing monitor lists
attrs = ['loss', 'err', 'acc']
monitors = OrderedDict()
for mtype in ['train', 'valid']:
monitors[mtype] = dict()
for attr in attrs:
monitors[mtype][attr] = list()
print('')
print('Train samples: {:8d}'.format(len(train[0])))
print('Valididation samples:{:8d}'.format(len(valid[0])))
print('Hidden layers: {:8d}'.format(config['hidden_layers']))
print('Hidden nodes/layer: {:8d}'.format(config['hidden_nodes']))
print('')
kwargs = {
'train': train,
'valid': valid,
'algo': config['algorithm'],
'learning_rate': config['learning_rate'],
'momentum': config['momentum'],
'hidden_l1': config['hidden_l1'],
'weight_l2': config['weight_l2'],
}
# Run with monitors if `plots` flag set to true
for t_monitors, v_monitors in net.itertrain(**kwargs):
for key in attrs:
monitors['train'][key].append(t_monitors[key])
monitors['valid'][key].append(v_monitors[key])
if plots == True:
plot_monitors(attrs, monitors['train'], monitors['valid'])
# Classify features against label/target value to get accuracy
# where `valid` is a tuple with validation (features, label)
accuracy = net.score(valid[0], valid[1])
return net, accuracy, monitors
trainingSetLabels = trainingSet[:, 12] #putting labels in separate array
trainingSetLabels[trainingSetLabels ==
0] = -1 #replacing all 0 with -1 to match sklearn format
trainingSet = trainingSet[:, 1:11] #removing label cols from actual inputs
trainingSet, testingSet, trainingSetLabels, testingSetLabels = train_test_split(
trainingSet, trainingSetLabels, test_size=0.6,
random_state=0) #fixes random_state so results reproducible
startTime = time.time()
print "Time before training = ", startTime
clf = theanets.Classifier(
(10, 5, 5, 5, 2)
) #dummy values for layers for now, 1 hidden layer -- 10 inputs mapped to a binary classification output (2 choices)
clf.itertrain(
[trainingSet, trainingSetLabels],
[testingSet, testingSetLabels],
algo=
'sgd', #theanets uses training/validation split to mean training/testing split, methinks
learning_rate=0.1,
momentum=0.9,
hidden_l1=0.000001, #sparse regularizer
input_noise=0.1,
hidden_noise=0.1,
input_dropout=0.3,
hidden_dropout=0.3, #Dropout and Noise regularizer to prevent overfitting
save_progress="TN1_model_save.txt",
save_every=1000,
def _build(self, *hiddens, **kwargs):
return theanets.Classifier(
layers=(self.DIGIT_SIZE, ) + hiddens + (10, ),
hidden_activation='logistic',
**kwargs)
def test_feed_forward(self):
net = theanets.Classifier((self.NUM_INPUTS, self.l))
out = net.predict_proba(self.INPUTS)
assert out.shape == (self.NUM_EXAMPLES, self.NUM_CLASSES)
def test_classification(self, loss):
net = theanets.Classifier([
self.NUM_INPUTS, 10, self.NUM_CLASSES], loss=loss)
self.assert_progress(net, 'sgd', [self.INPUTS, self.CLASSES])
def net(self):
return theanets.Classifier(u.CLF_LAYERS)
def build(self, *hiddens):
return theanets.Classifier(
[self.NUM_INPUTS] + list(hiddens) + [self.NUM_CLASSES],
weighted=True)
learner = oneNN(train[0], train[1])
oneclasses = np.apply_along_axis(learner, 1, test[0])
print "1-NN classifier!"
print "%s / %s " % (sum(oneclasses == test[1]), len(test[1]))
print theautil.classifications(oneclasses, test[1])
print '''
########################################################################
# Part 3. Let's start using neural networks!
########################################################################
'''
# try different combos here
net = theanets.Classifier([2, 3, 2])
net.train(train, valid, algo='layerwise', max_updates=mupdates, patience=1)
net.train(train, valid, algo='rprop', max_updates=mupdates, patience=1)
print "Learner on the test set"
classify = net.classify(test[0])
print "%s / %s " % (sum(classify == test[1]), len(test[1]))
print collections.Counter(classify)
print theautil.classifications(classify, test[1])
print net.layers[2].params[0].get_value()
print net.layers[2].params[0].get_value()
def real_function(pt):
rad = 0.1643167672515498
def test_classification(loss):
net = theanets.Classifier([
u.NUM_INPUTS, u.NUM_HID1, u.NUM_CLASSES], loss=loss)
u.assert_progress(net, u.CLF_DATA)
