def construct_dbn_from_stack(stack):
# some settings
irange = 0.05
layers = []
for ii, layer in enumerate(stack.layers()):
lr_scale = 0.25 if ii == 0 else 0.25
layers.append(
mlp.Sigmoid(dim=layer.nhid,
layer_name='h' + str(ii),
irange=irange,
W_lr_scale=lr_scale,
max_col_norm=2.))
# softmax layer at then end for classification
layers.append(
mlp.Softmax(n_classes=9,
layer_name='y',
irange=irange,
W_lr_scale=0.25))
dbn = mlp.MLP(layers=layers, nvis=stack.layers()[0].get_input_space().dim)
# copy weigths to DBN
for ii, layer in enumerate(stack.layers()):
dbn.layers[ii].set_weights(layer.get_weights())
dbn.layers[ii].set_biases(layer.hidbias.get_value(borrow=False))
return dbn
def __init__(self,
n_classes,
batch_size=None,
input_space=None,
irange=None,
istdev=None,
W_lr_scale=None,
b_lr_scale=None,
max_row_norm=None,
max_col_norm=None,
sparse_init=None,
init_bias_target_marginals=None,
nvis=None,
seed=None):
super(SoftmaxRegression, self).__init__(layers=[
mlp.Softmax(n_classes=n_classes,
layer_name='y',
irange=irange,
istdev=istdev,
sparse_init=sparse_init,
W_lr_scale=W_lr_scale,
b_lr_scale=b_lr_scale,
max_row_norm=max_row_norm,
max_col_norm=max_col_norm,
init_bias_target_marginals=init_bias_target_marginals)
],
batch_size=batch_size,
input_space=input_space,
nvis=nvis,
seed=seed)
def __init__(self,
n_classes,
batch_size=None,
input_space=None,
irange=None,
istdev=None,
W_lr_scale=None,
b_lr_scale=None,
max_row_norm=None,
max_col_norm=None,
sparse_init=None,
dropout_include_probs=None,
dropout_scales=None,
dropout_input_include_prob=None,
dropout_input_scale=None,
init_bias_target_marginals=None,
nvis=None,
seed=None):
"""
layers: a list of MLP_Layers. The final layer will specify the
MLP's output space.
batch_size: optional. if not None, then should be a positive
integer. Mostly useful if one of your layers
involves a theano op like convolution that requires
a hard-coded batch size.
input_space: a Space specifying the kind of input the MLP acts
on. If None, input space is specified by nvis.
See pylearn2.models.MLP for notes on dropout.
"""
super(SoftmaxRegression, self).__init__(
layers=[
mlp.Softmax(
n_classes=n_classes,
layer_name='y',
irange=irange,
istdev=istdev,
sparse_init=sparse_init,
W_lr_scale=W_lr_scale,
b_lr_scale=b_lr_scale,
max_row_norm=max_row_norm,
max_col_norm=max_col_norm,
init_bias_target_marginals=init_bias_target_marginals)
],
batch_size=batch_size,
input_space=input_space,
dropout_input_include_prob=dropout_input_include_prob,
dropout_input_scale=dropout_input_scale,
nvis=nvis,
seed=seed)
def _create_layer(self, name, layer, irange):
if isinstance(layer, Convolution):
return self._create_convolution_layer(name, layer, irange)
if layer.type == "Rectifier":
self._check_layer(layer, ['units'])
return mlp.RectifiedLinear(layer_name=name,
dim=layer.units,
irange=irange)
if layer.type == "Sigmoid":
self._check_layer(layer, ['units'])
return mlp.Sigmoid(layer_name=name, dim=layer.units, irange=irange)
if layer.type == "Tanh":
self._check_layer(layer, ['units'])
return mlp.Tanh(layer_name=name, dim=layer.units, irange=irange)
if layer.type == "Maxout":
self._check_layer(layer, ['units', 'pieces'])
return maxout.Maxout(layer_name=name,
num_units=layer.units,
num_pieces=layer.pieces,
irange=irange)
if layer.type == "Linear":
self._check_layer(layer, ['units'])
return mlp.Linear(layer_name=layer.name,
dim=layer.units,
irange=irange)
if layer.type == "Gaussian":
self._check_layer(layer, ['units'])
return mlp.LinearGaussian(layer_name=layer.name,
init_beta=0.1,
min_beta=0.001,
max_beta=1000,
beta_lr_scale=None,
dim=layer.units,
irange=irange)
if layer.type == "Softmax":
self._check_layer(layer, ['units'])
return mlp.Softmax(layer_name=layer.name,
n_classes=layer.units,
irange=irange)
# create datasets ds_train = the_data() ds_train, ds_valid = ds_train.split(0.7) ds_valid, ds_test = ds_valid.split(0.7) ##################################### #Define Model ##################################### # create sigmoid hidden layer with 20 nodes, init weights in range -0.05 to 0.05 and add # a bias with value 1 hidden_layer = mlp.Sigmoid(layer_name='h0', dim=1, irange=.05, init_bias=1.) # softmax output layer output_layer = mlp.Softmax(2, 'softmax', irange=.05) layers = [hidden_layer, output_layer] # create neural net ann = mlp.MLP(layers, nvis=ds_train.nr_inputs) ##################################### #Define Training ##################################### #L1 Weight Decay L1_cost = PL.costs.cost.SumOfCosts([PL.costs.cost.MethodCost(method='cost_from_X'), PL.costs.mlp.L1WeightDecay(coeffs=[0.1, 0.01])]) # momentum initial_momentum = .5 final_momentum = .99
from pylearn2.models import mlp
from pylearn2.training_algorithms import sgd
from pylearn2.termination_criteria import EpochCounter
from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
from csv_data import CSVData
import numpy as np
class MLPData(DenseDesignMatrix):
def __init__(self, X, y):
super(MLPData, self).__init__(X=X, y=y.astype(int), y_labels=2)
threshold = 0.95
hidden_layer = mlp.Sigmoid(layer_name='h0', dim=10, sparse_init=10)
output_layer = mlp.Softmax(layer_name='y', n_classes=2, irange=0.05)
layers = [hidden_layer, output_layer]
neural_net = mlp.MLP(layers, nvis=10)
trainer = sgd.SGD(batch_size=5,
learning_rate=.1,
termination_criterion=EpochCounter(100))
first = True
learning = True
correct = 0
incorrect = 0
total = 0
data = CSVData("results2.csv")
while True:
X, y = data.get_data()
if (X == None):
def train(d):
print 'Creating dataset'
# load mnist here
# X = d.train_X
# y = d.train_Y
# test_X = d.test_X
# test_Y = d.test_Y
# nb_classes = len(np.unique(y))
# train_y = convert_one_hot(y)
# train_set = DenseDesignMatrix(X=X, y=y)
train = DenseDesignMatrix(X=d.train_X, y=convert_one_hot(d.train_Y))
valid = DenseDesignMatrix(X=d.valid_X, y=convert_one_hot(d.valid_Y))
test = DenseDesignMatrix(X=d.test_X, y=convert_one_hot(d.test_Y))
print 'Setting up'
batch_size = 1000
conv = mlp.ConvRectifiedLinear(
layer_name='c0',
output_channels=20,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
# W_lr_scale=0.25,
max_kernel_norm=1.9365)
mout = MaxoutConvC01B(layer_name='m0',
num_pieces=4,
num_channels=96,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
W_lr_scale=0.25,
max_kernel_norm=1.9365)
mout2 = MaxoutConvC01B(layer_name='m1',
num_pieces=4,
num_channels=96,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
W_lr_scale=0.25,
max_kernel_norm=1.9365)
sigmoid = mlp.Sigmoid(
layer_name='Sigmoid',
dim=500,
sparse_init=15,
)
smax = mlp.Softmax(layer_name='y', n_classes=10, irange=0.)
in_space = Conv2DSpace(shape=[28, 28],
num_channels=1,
axes=['c', 0, 1, 'b'])
net = mlp.MLP(
layers=[mout, mout2, smax],
input_space=in_space,
# nvis=784,
)
trainer = bgd.BGD(batch_size=batch_size,
line_search_mode='exhaustive',
conjugate=1,
updates_per_batch=10,
monitoring_dataset={
'train': train,
'valid': valid,
'test': test
},
termination_criterion=termination_criteria.MonitorBased(
channel_name='valid_y_misclass'))
trainer = sgd.SGD(learning_rate=0.15,
cost=dropout.Dropout(),
batch_size=batch_size,
monitoring_dataset={
'train': train,
'valid': valid,
'test': test
},
termination_criterion=termination_criteria.MonitorBased(
channel_name='valid_y_misclass'))
trainer.setup(net, train)
epoch = 0
while True:
print 'Training...', epoch
trainer.train(dataset=train)
net.monitor()
epoch += 1
def main():
training_data, validation_data, test_data, std_scale = load_training_data()
kaggle_test_features = load_test_data(std_scale)
###############
# pylearn2 ML
hl1 = mlp.Sigmoid(layer_name='hl1', dim=200, irange=.1, init_bias=1.)
hl2 = mlp.Sigmoid(layer_name='hl2', dim=100, irange=.1, init_bias=1.)
# create Softmax output layer
output_layer = mlp.Softmax(9, 'output', irange=.1)
# create Stochastic Gradient Descent trainer that runs for 400 epochs
trainer = sgd.SGD(learning_rate=.05,
batch_size=300,
learning_rule=learning_rule.Momentum(.5),
termination_criterion=MonitorBased(
channel_name='valid_objective',
prop_decrease=0.,
N=10),
monitoring_dataset={
'valid': validation_data,
'train': training_data
})
layers = [hl1, hl2, output_layer]
# create neural net
model = mlp.MLP(layers, nvis=93)
watcher = best_params.MonitorBasedSaveBest(
channel_name='valid_objective',
save_path='pylearn2_results/pylearn2_test.pkl')
velocity = learning_rule.MomentumAdjustor(final_momentum=.6,
start=1,
saturate=250)
decay = sgd.LinearDecayOverEpoch(start=1, saturate=250, decay_factor=.01)
######################
experiment = Train(dataset=training_data,
model=model,
algorithm=trainer,
extensions=[watcher, velocity, decay])
experiment.main_loop()
#load best model and test
################
model = serial.load('pylearn2_results/pylearn2_test.pkl')
# get an prediction of the accuracy from the test_data
test_results = model.fprop(theano.shared(test_data[0],
name='test_data')).eval()
print test_results.shape
loss = multiclass_log_loss(test_data[1], test_results)
print 'Test multiclass log loss:', loss
out_file = 'pylearn2_results/' + str(loss) + 'ann'
#exp.save(out_file + '.pkl')
#save the kaggle results
results = model.fprop(
theano.shared(kaggle_test_features, name='kaggle_test_data')).eval()
save_results(out_file + '.csv', kaggle_test_features, results)
layer_name='l5',
#sparse_init=12,
irange=0.01,
dim=300,
#max_col_norm=1.
)
l6 = mlp.RectifiedLinear(
layer_name='l6',
#sparse_init=12,
irange=0.01,
dim=300,
#max_col_norm=1.
)
output = mlp.Softmax(n_classes=2, layer_name='y', irange=.01)
#output = mlp.HingeLoss(layer_name='y',n_classes=2,irange=.05)
#layers = [l5, l6, output]
layers = [l1, l2, l3, l4, l5, output]
ann = mlp.MLP(layers, nvis=X[0].reshape(-1).shape[0])
lr = 0.1
epochs = 400
trainer = sgd.SGD(
learning_rate=lr,
batch_size=100,
learning_rule=learning_rule.Momentum(.05),
# Remember, default dropout is .5
l5 = MaxoutConvC01B(layer_name='l5',
tied_b=1,
num_channels=256, num_pieces=2, pad=2,
kernel_shape=[3,3], pool_shape=[2,2], pool_stride=[2,2],
max_kernel_norm= 1.9365, irange=.025)
l6 = MaxoutConvC01B(layer_name='l6',
tied_b=1,
num_channels=256, num_pieces=2, pad=2,
kernel_shape=[3,3], pool_shape=[2,2], pool_stride=[2,2],
max_kernel_norm= 1.9365, irange=.025)
#dense layers
l7 = Maxout(layer_name='l7', num_units=1024, num_pieces=2, irange=.025)
l8 = Maxout(layer_name='l8', num_units=2048, num_pieces=2, irange=.025)
output_layer = mlp.Softmax(layer_name='y', n_classes=121, irange=.01)
layers = [l1,l2,l3,l4,l5, l6,l7, l8, output_layer]
images = []
y = []
file_names = []
dimensions = []
train_labels = [x for x in os.listdir("train") if os.path.isdir("{0}{1}{2}".format("train", os.sep, x))]
train_directories = ["{0}{1}{2}".format("train", os.sep, x) for x in train_labels]
train_labels, train_directories = zip(*sorted(zip(train_labels, train_directories), key=lambda x: x[0]))
for idx, folder in enumerate(train_directories):
for f_name_dir in os.walk(folder):
num_channels=192,
num_pieces=2,
kernel_shape=(5, 5),
pool_shape=(2, 2),
pool_stride=(2, 2),
irange=.005,
max_kernel_norm=1.9365)
l4 = maxout.Maxout(layer_name='l4',
irange=.005,
num_units=500,
num_pieces=5,
max_col_norm=1.9)
output = mlp.Softmax(layer_name='y',
n_classes=10,
irange=.005,
max_col_norm=1.9365)
layers = [l1, l2, l3, l4, output]
mdl = mlp.MLP(layers,
input_space=in_space)
trainer = sgd.SGD(learning_rate=.17,
batch_size=128,
learning_rule=learning_rule.Momentum(.5),
# Remember, default dropout is .5
cost=Dropout(input_include_probs={'l1': .8},
input_scales={'l1': 1.}),
termination_criterion=EpochCounter(max_epochs=475),
monitoring_dataset={'valid': tst,
ds = SHEHAD("/Users/evgeny/data/TRAIN")
vds = SHEHAD("/Users/evgeny/data/TEST")
hidden_layer = mlp.RectifiedLinear(layer_name='hidden',
dim=128,
irange=0.001,
init_bias=0)
hidden_layer2 = mlp.RectifiedLinear(layer_name='hidden2',
dim=128,
irange=0.01,
init_bias=0)
hidden_layer3 = mlp.RectifiedLinear(layer_name='hidden3',
dim=128,
irange=0.01,
init_bias=0)
# create Softmax output layer
output_layer = mlp.Softmax(3, 'output', irange=.1)
# create Stochastic Gradient Descent trainer that runs for 400 epochs
cost = NegativeLogLikelihoodCost()
rule = Momentum(0.9)
# rule = Momentum(0.9, True)
# update_callbacks=ExponentialDecay(1 + 1e-5, 0.001)
trainer = sgd.SGD(learning_rate=0.01,
cost=cost,
batch_size=128,
termination_criterion=EpochCounter(1000),
monitoring_dataset=vds,
learning_rule=rule)
layers = [hidden_layer, hidden_layer2, output_layer]
# create neural net that takes two inputs
ann = mlp.MLP(layers, nvis=ds.feat_cnt)
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
max_kernel_norm=1.9365)
layerh3 = mlp.ConvRectifiedLinear(layer_name='h3',
output_channels=64,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
max_kernel_norm=1.9365)
''' Note: changed the number of classes '''
layery = mlp.Softmax(max_col_norm=1.9365,
layer_name='y',
n_classes=121,
istdev=.05)
print 'Setting up trainers'
trainer = sgd.SGD(learning_rate=0.5,
batch_size=50,
termination_criterion=EpochCounter(200),
learning_rule=Momentum(init_momentum=0.5))
layers = [layerh2, layerh3, layery]
ann = mlp.MLP(layers, input_space=Conv2DSpace(shape=[28, 28], num_channels=1))
trainer.setup(ann, ds)
print 'Start Training'
while True:
trainer.train(dataset=ds)
ann.monitor.report_epoch()
ann.monitor()
if not trainer.continue_learning(ann):
def train(d=None):
train_X = np.array(d.train_X)
train_y = np.array(d.train_Y)
valid_X = np.array(d.valid_X)
valid_y = np.array(d.valid_Y)
test_X = np.array(d.test_X)
test_y = np.array(d.test_Y)
nb_classes = len(np.unique(train_y))
train_y = convert_one_hot(train_y)
valid_y = convert_one_hot(valid_y)
# train_set = RotationalDDM(X=train_X, y=train_y)
train_set = DenseDesignMatrix(X=train_X, y=train_y)
valid_set = DenseDesignMatrix(X=valid_X, y=valid_y)
print 'Setting up'
batch_size = 100
c0 = mlp.ConvRectifiedLinear(
layer_name='c0',
output_channels=64,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
# W_lr_scale=0.25,
max_kernel_norm=1.9365)
c1 = mlp.ConvRectifiedLinear(
layer_name='c1',
output_channels=64,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
# W_lr_scale=0.25,
max_kernel_norm=1.9365)
c2 = mlp.ConvRectifiedLinear(
layer_name='c2',
output_channels=64,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[5, 4],
W_lr_scale=0.25,
# max_kernel_norm=1.9365
)
sp0 = mlp.SoftmaxPool(
detector_layer_dim=16,
layer_name='sp0',
pool_size=4,
sparse_init=512,
)
sp1 = mlp.SoftmaxPool(
detector_layer_dim=16,
layer_name='sp1',
pool_size=4,
sparse_init=512,
)
r0 = mlp.RectifiedLinear(
layer_name='r0',
dim=512,
sparse_init=512,
)
r1 = mlp.RectifiedLinear(
layer_name='r1',
dim=512,
sparse_init=512,
)
s0 = mlp.Sigmoid(
layer_name='s0',
dim=500,
# max_col_norm=1.9365,
sparse_init=15,
)
out = mlp.Softmax(
n_classes=nb_classes,
layer_name='output',
irange=.0,
# max_col_norm=1.9365,
# sparse_init=nb_classes,
)
epochs = EpochCounter(100)
layers = [s0, out]
decay_coeffs = [.00005, .00005, .00005]
in_space = Conv2DSpace(
shape=[d.size, d.size],
num_channels=1,
)
vec_space = VectorSpace(d.size**2)
nn = mlp.MLP(
layers=layers,
# input_space=in_space,
nvis=d.size**2,
# batch_size=batch_size,
)
trainer = sgd.SGD(
learning_rate=0.01,
# cost=SumOfCosts(costs=[
# dropout.Dropout(),
# MethodCost(method='cost_from_X'),
# WeightDecay(decay_coeffs),
# ]),
# cost=MethodCost(method='cost_from_X'),
batch_size=batch_size,
# train_iteration_mode='even_shuffled_sequential',
termination_criterion=epochs,
# learning_rule=learning_rule.Momentum(init_momentum=0.5),
)
trainer = bgd.BGD(
batch_size=10000,
line_search_mode='exhaustive',
conjugate=1,
updates_per_batch=10,
termination_criterion=epochs,
)
lr_adjustor = LinearDecayOverEpoch(
start=1,
saturate=10,
decay_factor=.1,
)
momentum_adjustor = learning_rule.MomentumAdjustor(
final_momentum=.99,
start=1,
saturate=10,
)
trainer.setup(nn, train_set)
print 'Learning'
test_X = vec_space.np_format_as(test_X, nn.get_input_space())
train_X = vec_space.np_format_as(train_X, nn.get_input_space())
i = 0
X = nn.get_input_space().make_theano_batch()
Y = nn.fprop(X)
predict = theano.function([X], Y)
best = -40
best_iter = -1
while trainer.continue_learning(nn):
print '--------------'
print 'Training Epoch ' + str(i)
trainer.train(dataset=train_set)
nn.monitor()
print 'Evaluating...'
predictions = convert_categorical(predict(train_X[:2000]))
score = accuracy_score(convert_categorical(train_y[:2000]),
predictions)
print 'Score on train: ' + str(score)
predictions = convert_categorical(predict(test_X))
score = accuracy_score(test_y, predictions)
print 'Score on test: ' + str(score)
best, best_iter = (best, best_iter) if best > score else (score, i)
print 'Current best: ' + str(best) + ' at iter ' + str(best_iter)
print classification_report(test_y, predictions)
print 'Adjusting parameters...'
# momentum_adjustor.on_monitor(nn, valid_set, trainer)
# lr_adjustor.on_monitor(nn, valid_set, trainer)
i += 1
print ' '
def supervisedLayerwisePRL(trainset, testset):
'''
The supervised layerwise training as used in the PRL Paper.
Input
------
trainset : A path to an hdf5 file created through h5py.
testset : A path to an hdf5 file created through h5py.
'''
batch_size = 100
# Both train and test h5py files are expected to have a 'topo_view' and 'y'
# datasets side them corresponding to the 'b01c' data format as used in pylearn2
# and 'y' equivalent to the one hot encoded labels
trn = HDF5Dataset(filename=trainset,
topo_view='topo_view',
y='y',
load_all=False)
tst = HDF5Dataset(filename=testset,
topo_view='topo_view',
y='y',
load_all=False)
'''
The 1st Convolution and Pooling Layers are added below.
'''
h1 = mlp.ConvRectifiedLinear(layer_name='h1',
output_channels=64,
irange=0.05,
kernel_shape=[4, 4],
pool_shape=[4, 4],
pool_stride=[2, 2],
max_kernel_norm=1.9365)
fc = mlp.RectifiedLinear(layer_name='fc', dim=1500, irange=0.05)
output = mlp.Softmax(layer_name='y',
n_classes=171,
irange=.005,
max_col_norm=1.9365)
layers = [h1, fc, output]
mdl = mlp.MLP(layers,
input_space=Conv2DSpace(shape=(70, 70), num_channels=1))
trainer = sgd.SGD(
learning_rate=0.002,
batch_size=batch_size,
learning_rule=learning_rule.RMSProp(),
cost=SumOfCosts(
costs=[Default(),
WeightDecay(coeffs=[0.0005, 0.0005, 0.0005])]),
train_iteration_mode='shuffled_sequential',
monitor_iteration_mode='sequential',
termination_criterion=EpochCounter(max_epochs=15),
monitoring_dataset={
'test': tst,
'valid': vld
})
watcher = best_params.MonitorBasedSaveBest(
channel_name='valid_y_misclass',
save_path='./Saved Models/conv_supervised_layerwise_best1.pkl')
decay = sgd.LinearDecayOverEpoch(start=8, saturate=15, decay_factor=0.1)
experiment = Train(
dataset=trn,
model=mdl,
algorithm=trainer,
extensions=[watcher, decay],
)
experiment.main_loop()
del mdl
mdl = serial.load('./Saved Models/conv_supervised_layerwise_best1.pkl')
mdl = push_monitor(mdl, 'k')
'''
The 2nd Convolution and Pooling Layers are added below.
'''
h2 = mlp.ConvRectifiedLinear(layer_name='h2',
output_channels=64,
irange=0.05,
kernel_shape=[4, 4],
pool_shape=[4, 4],
pool_stride=[2, 2],
max_kernel_norm=1.9365)
fc = mlp.RectifiedLinear(layer_name='fc', dim=1500, irange=0.05)
output = mlp.Softmax(layer_name='y',
n_classes=171,
irange=.005,
max_col_norm=1.9365)
del mdl.layers[-1]
mdl.layer_names.remove('y')
del mdl.layers[-1]
mdl.layer_names.remove('fc')
mdl.add_layers([h2, fc, output])
trainer = sgd.SGD(learning_rate=0.002,
batch_size=batch_size,
learning_rule=learning_rule.RMSProp(),
cost=SumOfCosts(costs=[
Default(),
WeightDecay(coeffs=[0.0005, 0.0005, 0.0005, 0.0005])
]),
train_iteration_mode='shuffled_sequential',
monitor_iteration_mode='sequential',
termination_criterion=EpochCounter(max_epochs=15),
monitoring_dataset={
'test': tst,
'valid': vld
})
watcher = best_params.MonitorBasedSaveBest(
channel_name='valid_y_misclass',
save_path='./Saved Models/conv_supervised_layerwise_best2.pkl')
decay = sgd.LinearDecayOverEpoch(start=8, saturate=15, decay_factor=0.1)
experiment = Train(
dataset=trn,
model=mdl,
algorithm=trainer,
extensions=[watcher, decay],
)
experiment.main_loop()
del mdl
mdl = serial.load('./Saved Models/conv_supervised_layerwise_best2.pkl')
mdl = push_monitor(mdl, 'l')
'''
The 3rd Convolution and Pooling Layers are added below.
'''
h3 = mlp.ConvRectifiedLinear(layer_name='h2',
output_channels=64,
irange=0.05,
kernel_shape=[4, 4],
pool_shape=[4, 4],
pool_stride=[2, 2],
max_kernel_norm=1.9365)
fc = mlp.RectifiedLinear(layer_name='h3', dim=1500, irange=0.05)
output = mlp.Softmax(layer_name='y',
n_classes=10,
irange=.005,
max_col_norm=1.9365)
del mdl.layers[-1]
mdl.layer_names.remove('y')
del mdl.layers[-1]
mdl.layer_names.remove('fc')
mdl.add_layers([h3, output])
trainer = sgd.SGD(
learning_rate=.002,
batch_size=batch_size,
learning_rule=learning_rule.RMSProp(),
cost=SumOfCosts(costs=[
Default(),
WeightDecay(coeffs=[0.0005, 0.0005, 0.0005, 0.0005, 0.0005])
]),
train_iteration_mode='shuffled_sequential',
monitor_iteration_mode='sequential',
termination_criterion=EpochCounter(max_epochs=15),
monitoring_dataset={
'test': tst,
'valid': vld
})
watcher = best_params.MonitorBasedSaveBest(
channel_name='valid_y_misclass',
save_path='./Saved Models/conv_supervised_layerwise_best3.pkl')
decay = sgd.LinearDecayOverEpoch(start=8, saturate=15, decay_factor=0.1)
experiment = Train(
dataset=trn,
model=mdl,
algorithm=trainer,
extensions=[watcher, decay],
)
experiment.main_loop()
expect_labels=True,
expect_headers=False,
delimiter=',')
valid = csv_dataset.CSVDataset("../data/valid.csv",
expect_labels=True,
expect_headers=False,
delimiter=',')
test = csv_dataset.CSVDataset("../data/test.csv",
expect_labels=True,
expect_headers=False,
delimiter=',')
# ------------------------------------------Simple ANN
h0 = mlp.Sigmoid(layer_name="h0", dim=73, sparse_init=0)
y0 = mlp.Softmax(n_classes=5, layer_name="y0", irange=0)
layers = [h0, y0]
nn = mlp.MLP(layers, nvis=train.X.shape[1])
algo = sgd.SGD(learning_rate=0.05,
batch_size=100,
monitoring_dataset=valid,
termination_criterion=EpochCounter(100))
algo.setup(nn, train)
save_best = best_params.MonitorBasedSaveBest(channel_name="objective",
save_path='best_params.pkl')
while True:
algo.train(dataset=train)
nn.monitor.report_epoch()
nn.monitor()
