def runSP():
ds = StockPrice()
# create hidden layer with 2 nodes, init weights in range -0.1 to 0.1 and add
# a bias with value 1
hidden_layer = mlp.Sigmoid(layer_name='hidden', dim=10000, irange=.1, init_bias=1.)
# create Softmax output layer
output_layer = mlp.Linear(layer_name='output', dim=1, irange=.1, init_bias=1.)
# create Stochastic Gradient Descent trainer that runs for 400 epochs
trainer = sgd.SGD(learning_rate=.005, batch_size=500, termination_criterion=EpochCounter(10))
layers = [hidden_layer, output_layer]
# create neural net that takes two inputs
ann = mlp.MLP(layers, nvis=1000)
trainer.setup(ann, ds)
# train neural net until the termination criterion is true
while True:
trainer.train(dataset=ds)
ann.monitor.report_epoch()
ann.monitor()
if not trainer.continue_learning(ann):
break
#accuracy = Accuracy()
acc = Accuracy()
for i, predict in enumerate(ann.fprop(theano.shared(ds.valid[0], name='inputs')).eval()):
print predict, ds.valid[1][i]
acc.evaluatePN(predict[0], ds.valid[1][i][0])
acc.printResult()
def construct_dbn_from_stack(stack):
# some settings
irange = 0.05
layers = []
for ii, layer in enumerate(stack.layers()):
lr_scale = 0.25 if ii == 0 else 0.25
layers.append(
mlp.Sigmoid(dim=layer.nhid,
layer_name='h' + str(ii),
irange=irange,
W_lr_scale=lr_scale,
max_col_norm=2.))
# softmax layer at then end for classification
layers.append(
mlp.Softmax(n_classes=9,
layer_name='y',
irange=irange,
W_lr_scale=0.25))
dbn = mlp.MLP(layers=layers, nvis=stack.layers()[0].get_input_space().dim)
# copy weigths to DBN
for ii, layer in enumerate(stack.layers()):
dbn.layers[ii].set_weights(layer.get_weights())
dbn.layers[ii].set_biases(layer.hidbias.get_value(borrow=False))
return dbn
def _create_layer(self, name, layer, irange):
if isinstance(layer, Convolution):
return self._create_convolution_layer(name, layer, irange)
if layer.type == "Rectifier":
self._check_layer(layer, ['units'])
return mlp.RectifiedLinear(layer_name=name,
dim=layer.units,
irange=irange)
if layer.type == "Sigmoid":
self._check_layer(layer, ['units'])
return mlp.Sigmoid(layer_name=name, dim=layer.units, irange=irange)
if layer.type == "Tanh":
self._check_layer(layer, ['units'])
return mlp.Tanh(layer_name=name, dim=layer.units, irange=irange)
if layer.type == "Maxout":
self._check_layer(layer, ['units', 'pieces'])
return maxout.Maxout(layer_name=name,
num_units=layer.units,
num_pieces=layer.pieces,
irange=irange)
if layer.type == "Linear":
self._check_layer(layer, ['units'])
return mlp.Linear(layer_name=layer.name,
dim=layer.units,
irange=irange)
if layer.type == "Gaussian":
self._check_layer(layer, ['units'])
return mlp.LinearGaussian(layer_name=layer.name,
init_beta=0.1,
min_beta=0.001,
max_beta=1000,
beta_lr_scale=None,
dim=layer.units,
irange=irange)
if layer.type == "Softmax":
self._check_layer(layer, ['units'])
return mlp.Softmax(layer_name=layer.name,
n_classes=layer.units,
irange=irange)
def __init__(self, data):
self.N = 5 * 5
self.predictionLength = 2
# create hidden layer with 2 nodes, init weights in range -0.1 to 0.1 and add
# a bias with value 1
hidden_layer = mlp.Sigmoid(layer_name='hidden',
dim=25,
irange=.1,
init_bias=1.)
# create Linear output layer
output_layer = mlp.Linear(1, 'output', irange=.1, init_bias=1.)
# create Stochastic Gradient Descent trainer that runs for 400 epochs
trainer = sgd.SGD(learning_rate=.005,
batch_size=10,
termination_criterion=EpochCounter(100))
layers = [hidden_layer, output_layer]
# create neural net that takes two inputs
nn = mlp.MLP(layers, nvis=self.N)
NeuralNetwork.__init__(self, data, nn, trainer)
def runXOR():
ds = XOR()
hidden_layer = mlp.Sigmoid(layer_name='hidden', dim=10, irange=.1, init_bias=1.)
output_layer = mlp.Linear(layer_name='output', dim=1, irange=.1, init_bias=1.)
trainer = sgd.SGD(learning_rate=.05, batch_size=1, termination_criterion=EpochCounter(1000))
layers = [hidden_layer, output_layer]
# create neural net that takes two inputs
ann = mlp.MLP(layers, nvis=4)
trainer.setup(ann, ds)
# train neural net until the termination criterion is true
while True:
trainer.train(dataset=ds)
#ann.monitor.report_epoch()
#ann.monitor()
if not trainer.continue_learning(ann):
break
inputs= np.array([[0, 0, 0, 1]])
print ann.fprop(theano.shared(inputs, name='inputs')).eval()
inputs = np.array([[0, 1, 0, 1]])
print ann.fprop(theano.shared(inputs, name='inputs')).eval()
inputs = np.array([[1, 1, 1, 1]])
print ann.fprop(theano.shared(inputs, name='inputs')).eval()
inputs = np.array([[1, 1, 0, 0]])
print ann.fprop(theano.shared(inputs, name='inputs')).eval()
def init_train():
# Initialize train object.
idpath = os.path.splitext(
os.path.abspath(__file__))[0] # ID for output files.
save_path = idpath + '.pkl'
# Dataset
#seed = 42
benchmark = 1
#derived_feat, nvis = False, 21
derived_feat, nvis = True, 28
#derived_feat, nvis = 'only', 7
dataset_train = pylearn2.datasets.physics.PHYSICS(
which_set='train', benchmark=benchmark, derived_feat=derived_feat)
dataset_train_monitor = pylearn2.datasets.physics.PHYSICS(
which_set='train',
benchmark=benchmark,
derived_feat=derived_feat,
start=0,
stop=100000)
dataset_valid = pylearn2.datasets.physics.PHYSICS(
which_set='valid', benchmark=benchmark, derived_feat=derived_feat)
dataset_test = pylearn2.datasets.physics.PHYSICS(which_set='test',
benchmark=benchmark,
derived_feat=derived_feat)
# Parameters
momentum_saturate = 200
# Model
model = pylearn2.models.mlp.MLP(
layers=[
mlp.Tanh(layer_name='h0', dim=300, istdev=.1),
#istdev=.05 for any intermediates
mlp.Sigmoid(layer_name='y', dim=1, istdev=.001)
],
nvis=nvis)
# Algorithm
algorithm = pylearn2.training_algorithms.sgd.SGD(
batch_size=100, # If changed, change learning rate!
learning_rate=
.05, # In dropout paper=10 for gradient averaged over batch. Depends on batchsize.
init_momentum=.9,
monitoring_dataset={
'train': dataset_train_monitor,
'valid': dataset_valid,
'test': dataset_test
},
termination_criterion=pylearn2.termination_criteria.Or(criteria=[
pylearn2.termination_criteria.MonitorBased(
channel_name="valid_objective", prop_decrease=0.00001, N=10),
pylearn2.termination_criteria.EpochCounter(
max_epochs=momentum_saturate)
]),
cost=pylearn2.costs.cost.SumOfCosts(costs=[
pylearn2.costs.mlp.Default(),
pylearn2.costs.mlp.WeightDecay(coeffs=[.00001, .00001])
]),
update_callbacks=pylearn2.training_algorithms.sgd.ExponentialDecay(
decay_factor=
1.0000002, # Decreases by this factor every batch. (1/(1.000001^8000)^100
min_lr=.000001))
# Extensions
extensions = [
#pylearn2.train_extensions.best_params.MonitorBasedSaveBest(channel_name='train_y_misclass',save_path=save_path)
pylearn2.training_algorithms.sgd.MomentumAdjustor(
start=0,
saturate=momentum_saturate,
final_momentum=.99 # Dropout=.5->.99 over 500 epochs.
)
]
# Train
train = pylearn2.train.Train(dataset=dataset_train,
model=model,
algorithm=algorithm,
extensions=extensions,
save_path=save_path,
save_freq=100)
return train
def __linit(self, X, y):
if (self.verbose > 0):
print "Lazy initialisation"
layers = self.layers
pylearn2mlp_layers = []
self.units_per_layer = []
#input layer units
self.units_per_layer += [X.shape[1]]
for layer in layers[:-1]:
self.units_per_layer += [layer[1]]
#Output layer units
self.units_per_layer += [y.shape[1]]
if (self.verbose > 0):
print "Units per layer", str(self.units_per_layer)
for i, layer in enumerate(layers[:-1]):
fan_in = self.units_per_layer[i] + 1
fan_out = self.units_per_layer[i + 1]
lim = np.sqrt(6) / (np.sqrt(fan_in + fan_out))
layer_name = "Hidden_%i_%s" % (i, layer[0])
activate_type = layer[0]
if activate_type == "RectifiedLinear":
hidden_layer = mlp.RectifiedLinear(dim=layer[1],
layer_name=layer_name,
irange=lim)
elif activate_type == "Sigmoid":
hidden_layer = mlp.Sigmoid(dim=layer[1],
layer_name=layer_name,
irange=lim)
elif activate_type == "Tanh":
hidden_layer = mlp.Tanh(dim=layer[1],
layer_name=layer_name,
irange=lim)
elif activate_type == "Maxout":
hidden_layer = maxout.Maxout(num_units=layer[1],
num_pieces=layer[2],
layer_name=layer_name,
irange=lim)
else:
raise NotImplementedError(
"Layer of type %s are not implemented yet" % layer[0])
pylearn2mlp_layers += [hidden_layer]
output_layer_info = layers[-1]
output_layer_name = "Output_%s" % output_layer_info[0]
fan_in = self.units_per_layer[-2] + 1
fan_out = self.units_per_layer[-1]
lim = np.sqrt(6) / (np.sqrt(fan_in + fan_out))
if (output_layer_info[0] == "Linear"):
output_layer = mlp.Linear(dim=self.units_per_layer[-1],
layer_name=output_layer_name,
irange=lim)
pylearn2mlp_layers += [output_layer]
self.mlp = mlp.MLP(pylearn2mlp_layers, nvis=self.units_per_layer[0])
self.ds = DenseDesignMatrix(X=X, y=y)
self.trainer.setup(self.mlp, self.ds)
inputs = self.mlp.get_input_space().make_theano_batch()
self.f = theano.function([inputs], self.mlp.fprop(inputs))
def train(d):
print 'Creating dataset'
# load mnist here
# X = d.train_X
# y = d.train_Y
# test_X = d.test_X
# test_Y = d.test_Y
# nb_classes = len(np.unique(y))
# train_y = convert_one_hot(y)
# train_set = DenseDesignMatrix(X=X, y=y)
train = DenseDesignMatrix(X=d.train_X, y=convert_one_hot(d.train_Y))
valid = DenseDesignMatrix(X=d.valid_X, y=convert_one_hot(d.valid_Y))
test = DenseDesignMatrix(X=d.test_X, y=convert_one_hot(d.test_Y))
print 'Setting up'
batch_size = 1000
conv = mlp.ConvRectifiedLinear(
layer_name='c0',
output_channels=20,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
# W_lr_scale=0.25,
max_kernel_norm=1.9365)
mout = MaxoutConvC01B(layer_name='m0',
num_pieces=4,
num_channels=96,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
W_lr_scale=0.25,
max_kernel_norm=1.9365)
mout2 = MaxoutConvC01B(layer_name='m1',
num_pieces=4,
num_channels=96,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
W_lr_scale=0.25,
max_kernel_norm=1.9365)
sigmoid = mlp.Sigmoid(
layer_name='Sigmoid',
dim=500,
sparse_init=15,
)
smax = mlp.Softmax(layer_name='y', n_classes=10, irange=0.)
in_space = Conv2DSpace(shape=[28, 28],
num_channels=1,
axes=['c', 0, 1, 'b'])
net = mlp.MLP(
layers=[mout, mout2, smax],
input_space=in_space,
# nvis=784,
)
trainer = bgd.BGD(batch_size=batch_size,
line_search_mode='exhaustive',
conjugate=1,
updates_per_batch=10,
monitoring_dataset={
'train': train,
'valid': valid,
'test': test
},
termination_criterion=termination_criteria.MonitorBased(
channel_name='valid_y_misclass'))
trainer = sgd.SGD(learning_rate=0.15,
cost=dropout.Dropout(),
batch_size=batch_size,
monitoring_dataset={
'train': train,
'valid': valid,
'test': test
},
termination_criterion=termination_criteria.MonitorBased(
channel_name='valid_y_misclass'))
trainer.setup(net, train)
epoch = 0
while True:
print 'Training...', epoch
trainer.train(dataset=train)
net.monitor()
epoch += 1
def main():
training_data, validation_data, test_data, std_scale = load_training_data()
kaggle_test_features = load_test_data(std_scale)
###############
# pylearn2 ML
hl1 = mlp.Sigmoid(layer_name='hl1', dim=200, irange=.1, init_bias=1.)
hl2 = mlp.Sigmoid(layer_name='hl2', dim=100, irange=.1, init_bias=1.)
# create Softmax output layer
output_layer = mlp.Softmax(9, 'output', irange=.1)
# create Stochastic Gradient Descent trainer that runs for 400 epochs
trainer = sgd.SGD(learning_rate=.05,
batch_size=300,
learning_rule=learning_rule.Momentum(.5),
termination_criterion=MonitorBased(
channel_name='valid_objective',
prop_decrease=0.,
N=10),
monitoring_dataset={
'valid': validation_data,
'train': training_data
})
layers = [hl1, hl2, output_layer]
# create neural net
model = mlp.MLP(layers, nvis=93)
watcher = best_params.MonitorBasedSaveBest(
channel_name='valid_objective',
save_path='pylearn2_results/pylearn2_test.pkl')
velocity = learning_rule.MomentumAdjustor(final_momentum=.6,
start=1,
saturate=250)
decay = sgd.LinearDecayOverEpoch(start=1, saturate=250, decay_factor=.01)
######################
experiment = Train(dataset=training_data,
model=model,
algorithm=trainer,
extensions=[watcher, velocity, decay])
experiment.main_loop()
#load best model and test
################
model = serial.load('pylearn2_results/pylearn2_test.pkl')
# get an prediction of the accuracy from the test_data
test_results = model.fprop(theano.shared(test_data[0],
name='test_data')).eval()
print test_results.shape
loss = multiclass_log_loss(test_data[1], test_results)
print 'Test multiclass log loss:', loss
out_file = 'pylearn2_results/' + str(loss) + 'ann'
#exp.save(out_file + '.pkl')
#save the kaggle results
results = model.fprop(
theano.shared(kaggle_test_features, name='kaggle_test_data')).eval()
save_results(out_file + '.csv', kaggle_test_features, results)
from pylearn2.training_algorithms.sgd import ExponentialDecay;
from deepmath.intfactor_dataset import IFDataset;
train_data=IFDataset(path="${PYLEARN2_DATA_PATH}/deepmath/factorize_data.pkl");
valid_data=IFDataset(path="${PYLEARN2_DATA_PATH}/deepmath/factorize_data.pkl",
which_set="valid");
test_data=IFDataset(path="${PYLEARN2_DATA_PATH}/deepmath/factorize_data.pkl",
which_set="test");
print "[MESSAGE] The datasets are loaded";
### build model
model=MLP(layers=[mlp.Sigmoid(layer_name="hidden_0",
dim=300,
istdev=0.1),
mlp.Sigmoid(layer_name="hidden_1",
dim=300,
istdev=0.1),
mlp.Sigmoid(layer_name="y",
dim=13,
istdev=0.01,
monitor_style="bit_vector_class")],
nvis=26);
print "[MESSAGE] The model is built";
### build algorithm
algorithm=SGD(batch_size=100,
train = csv_dataset.CSVDataset("../data/train.csv",
expect_labels=True,
expect_headers=False,
delimiter=',')
valid = csv_dataset.CSVDataset("../data/valid.csv",
expect_labels=True,
expect_headers=False,
delimiter=',')
test = csv_dataset.CSVDataset("../data/test.csv",
expect_labels=True,
expect_headers=False,
delimiter=',')
# ------------------------------------------Simple ANN
h0 = mlp.Sigmoid(layer_name="h0", dim=73, sparse_init=0)
y0 = mlp.Softmax(n_classes=5, layer_name="y0", irange=0)
layers = [h0, y0]
nn = mlp.MLP(layers, nvis=train.X.shape[1])
algo = sgd.SGD(learning_rate=0.05,
batch_size=100,
monitoring_dataset=valid,
termination_criterion=EpochCounter(100))
algo.setup(nn, train)
save_best = best_params.MonitorBasedSaveBest(channel_name="objective",
save_path='best_params.pkl')
while True:
algo.train(dataset=train)
nn.monitor.report_epoch()
def init_train(args):
# Interpret arguments.
derived_feat, seed, nhid, width, lrinit, lrdecay, momentum_init, momentum_saturate, wdecay, dropout_include, = args
derived_feat = derived_feat # string
seed = int(seed)
nhid = int(nhid)
width = int(width)
lrinit = float(lrinit)
lrdecay = float(lrdecay)
momentum_init = float(momentum_init)
momentum_saturate = int(momentum_saturate)
wdecay = float(wdecay)
dropout_include = float(
dropout_include) # dropout include probability in top layer.
# Specify output files: log and saved model pkl.
#idpath = os.path.splitext(os.path.abspath(__file__))[0] # ID for output files.
idpath = ''
idpath = idpath + '%s_%d_%d_%d_%0.5f_%0.9f_%0.2f_%d_%f_%0.1f' % (
derived_feat, seed, nhid, width, lrinit, lrdecay, momentum_init,
momentum_saturate, wdecay, dropout_include)
save_path = idpath + '.pkl'
logfile = idpath + '.log'
print 'Using=%s' % theano.config.device # Can use gpus.
print 'Writing to %s' % logfile
print 'Writing to %s' % save_path
sys.stdout = open(logfile, 'w')
# Dataset
benchmark = 1
dataset_train = pylearn2.datasets.physics.PHYSICS(
which_set='train', benchmark=benchmark, derived_feat=derived_feat)
#dataset_train = pylearn2.datasets.physics.PHYSICS(which_set='train', benchmark=benchmark, derived_feat=derived_feat, start=0, stop=2600000) # Smaller set for choosing hyperparameters.
dataset_train_monitor = pylearn2.datasets.physics.PHYSICS(
which_set='train',
benchmark=benchmark,
derived_feat=derived_feat,
start=0,
stop=100000)
dataset_valid = pylearn2.datasets.physics.PHYSICS(
which_set='valid', benchmark=benchmark, derived_feat=derived_feat)
dataset_test = pylearn2.datasets.physics.PHYSICS(which_set='test',
benchmark=benchmark,
derived_feat=derived_feat)
# Model
nvis = dataset_train.X.shape[1]
istdev = 1.0 / np.sqrt(width)
layers = []
for i in range(nhid):
# Hidden layer i
layer = mlp.Tanh(
layer_name='h%d' % i,
dim=width,
istdev=(istdev if i > 0 else
0.1), # First layer should have higher stdev.
)
layers.append(layer)
#layers.append(mlp.Sigmoid(layer_name='y', dim=1, istdev=istdev/100.0))
layers.append(mlp.Sigmoid(layer_name='y', dim=1, istdev=0.001))
model = pylearn2.models.mlp.MLP(layers, nvis=nvis, seed=seed)
# Cost
cost = pylearn2.costs.mlp.Default() # Default cost.
if dropout_include != 1.0:
# Use dropout cost if specified.
cost = pylearn2.costs.mlp.dropout.Dropout(
input_include_probs={'y': dropout_include},
input_scales={'y': 1.0 / dropout_include},
default_input_include_prob=1.0,
default_input_scale=1.0)
if wdecay != 0.0:
# Add weight decay term if specified.
cost2 = pylearn2.costs.mlp.WeightDecay(
coeffs=[wdecay] * (nhid + 1)) # wdecay if specified.
cost = pylearn2.costs.cost.SumOfCosts(costs=[cost, cost2])
# Algorithm
algorithm = pylearn2.training_algorithms.sgd.SGD(
batch_size=100, # If changed, change learning rate!
learning_rate=
lrinit, #.05, # In dropout paper=10 for gradient averaged over batch. Depends on batchsize.
learning_rule=pylearn2.training_algorithms.learning_rule.Momentum(
init_momentum=momentum_init, ),
monitoring_dataset={
'train': dataset_train_monitor,
'valid': dataset_valid,
'test': dataset_test
},
termination_criterion=pylearn2.termination_criteria.Or(criteria=[
pylearn2.termination_criteria.MonitorBased(
channel_name="valid_y_kl", prop_decrease=0.00001, N=10),
pylearn2.termination_criteria.EpochCounter(
max_epochs=momentum_saturate)
]),
cost=cost,
update_callbacks=pylearn2.training_algorithms.sgd.ExponentialDecay(
decay_factor=
lrdecay, #1.0000002 # Decreases by this factor every batch. (1/(1.000001^8000)^100
min_lr=.000001))
# Extensions
extensions = [
#pylearn2.train_extensions.best_params.MonitorBasedSaveBest(channel_name='train_y_misclass',save_path=save_path)
pylearn2.training_algorithms.learning_rule.MomentumAdjustor(
start=0,
saturate=momentum_saturate, # 200,500
final_momentum=0.99, # Dropout=.5->.99 over 500 epochs.
)
]
# Train
train = pylearn2.train.Train(dataset=dataset_train,
model=model,
algorithm=algorithm,
extensions=extensions,
save_path=save_path,
save_freq=100)
return train
def train(d=None):
train_X = np.array(d.train_X)
train_y = np.array(d.train_Y)
valid_X = np.array(d.valid_X)
valid_y = np.array(d.valid_Y)
test_X = np.array(d.test_X)
test_y = np.array(d.test_Y)
nb_classes = len(np.unique(train_y))
train_y = convert_one_hot(train_y)
valid_y = convert_one_hot(valid_y)
# train_set = RotationalDDM(X=train_X, y=train_y)
train_set = DenseDesignMatrix(X=train_X, y=train_y)
valid_set = DenseDesignMatrix(X=valid_X, y=valid_y)
print 'Setting up'
batch_size = 100
c0 = mlp.ConvRectifiedLinear(
layer_name='c0',
output_channels=64,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
# W_lr_scale=0.25,
max_kernel_norm=1.9365)
c1 = mlp.ConvRectifiedLinear(
layer_name='c1',
output_channels=64,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[2, 2],
# W_lr_scale=0.25,
max_kernel_norm=1.9365)
c2 = mlp.ConvRectifiedLinear(
layer_name='c2',
output_channels=64,
irange=.05,
kernel_shape=[5, 5],
pool_shape=[4, 4],
pool_stride=[5, 4],
W_lr_scale=0.25,
# max_kernel_norm=1.9365
)
sp0 = mlp.SoftmaxPool(
detector_layer_dim=16,
layer_name='sp0',
pool_size=4,
sparse_init=512,
)
sp1 = mlp.SoftmaxPool(
detector_layer_dim=16,
layer_name='sp1',
pool_size=4,
sparse_init=512,
)
r0 = mlp.RectifiedLinear(
layer_name='r0',
dim=512,
sparse_init=512,
)
r1 = mlp.RectifiedLinear(
layer_name='r1',
dim=512,
sparse_init=512,
)
s0 = mlp.Sigmoid(
layer_name='s0',
dim=500,
# max_col_norm=1.9365,
sparse_init=15,
)
out = mlp.Softmax(
n_classes=nb_classes,
layer_name='output',
irange=.0,
# max_col_norm=1.9365,
# sparse_init=nb_classes,
)
epochs = EpochCounter(100)
layers = [s0, out]
decay_coeffs = [.00005, .00005, .00005]
in_space = Conv2DSpace(
shape=[d.size, d.size],
num_channels=1,
)
vec_space = VectorSpace(d.size**2)
nn = mlp.MLP(
layers=layers,
# input_space=in_space,
nvis=d.size**2,
# batch_size=batch_size,
)
trainer = sgd.SGD(
learning_rate=0.01,
# cost=SumOfCosts(costs=[
# dropout.Dropout(),
# MethodCost(method='cost_from_X'),
# WeightDecay(decay_coeffs),
# ]),
# cost=MethodCost(method='cost_from_X'),
batch_size=batch_size,
# train_iteration_mode='even_shuffled_sequential',
termination_criterion=epochs,
# learning_rule=learning_rule.Momentum(init_momentum=0.5),
)
trainer = bgd.BGD(
batch_size=10000,
line_search_mode='exhaustive',
conjugate=1,
updates_per_batch=10,
termination_criterion=epochs,
)
lr_adjustor = LinearDecayOverEpoch(
start=1,
saturate=10,
decay_factor=.1,
)
momentum_adjustor = learning_rule.MomentumAdjustor(
final_momentum=.99,
start=1,
saturate=10,
)
trainer.setup(nn, train_set)
print 'Learning'
test_X = vec_space.np_format_as(test_X, nn.get_input_space())
train_X = vec_space.np_format_as(train_X, nn.get_input_space())
i = 0
X = nn.get_input_space().make_theano_batch()
Y = nn.fprop(X)
predict = theano.function([X], Y)
best = -40
best_iter = -1
while trainer.continue_learning(nn):
print '--------------'
print 'Training Epoch ' + str(i)
trainer.train(dataset=train_set)
nn.monitor()
print 'Evaluating...'
predictions = convert_categorical(predict(train_X[:2000]))
score = accuracy_score(convert_categorical(train_y[:2000]),
predictions)
print 'Score on train: ' + str(score)
predictions = convert_categorical(predict(test_X))
score = accuracy_score(test_y, predictions)
print 'Score on test: ' + str(score)
best, best_iter = (best, best_iter) if best > score else (score, i)
print 'Current best: ' + str(best) + ' at iter ' + str(best_iter)
print classification_report(test_y, predictions)
print 'Adjusting parameters...'
# momentum_adjustor.on_monitor(nn, valid_set, trainer)
# lr_adjustor.on_monitor(nn, valid_set, trainer)
i += 1
print ' '
import theano
from pylearn2.models import mlp
from pylearn2.training_algorithms import sgd
from pylearn2.termination_criteria import EpochCounter
from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
from csv_data import CSVData
import numpy as np
class MLPData(DenseDesignMatrix):
def __init__(self, X, y):
super(MLPData, self).__init__(X=X, y=y.astype(int), y_labels=2)
threshold = 0.95
hidden_layer = mlp.Sigmoid(layer_name='h0', dim=10, sparse_init=10)
output_layer = mlp.Softmax(layer_name='y', n_classes=2, irange=0.05)
layers = [hidden_layer, output_layer]
neural_net = mlp.MLP(layers, nvis=10)
trainer = sgd.SGD(batch_size=5,
learning_rate=.1,
termination_criterion=EpochCounter(100))
first = True
learning = True
correct = 0
incorrect = 0
total = 0
data = CSVData("results2.csv")
while True:
X, y = data.get_data()
y = []
for a, b in X:
if a + b == 1:
y.append([0, 1])
else:
y.append([1, 0])
X = np.array(X)
y = np.array(y)
super(XOR, self).__init__(X=X, y=y)
# create XOR dataset
ds = XOR()
# create hidden layer with 2 nodes, init weights in range -0.1 to 0.1 and add
# a bias with value 1
hidden_layer = mlp.Sigmoid(layer_name='hidden', dim=2, irange=.1, init_bias=1.)
# create Softmax output layer
output_layer = mlp.Softmax(2, 'output', irange=.1)
# create Stochastic Gradient Descent trainer that runs for 400 epochs
trainer = sgd.SGD(learning_rate=.05,
batch_size=10,
termination_criterion=EpochCounter(400))
layers = [hidden_layer, output_layer]
# create neural net that takes two inputs
ann = mlp.MLP(layers, nvis=2)
trainer.setup(ann, ds)
# train neural net until the termination criterion is true
while True:
trainer.train(dataset=ds)
ann.monitor.report_epoch()
ann.monitor()
return itertools.izip_longest(self.X, self.y)
# create datasets
ds_train = the_data()
ds_train, ds_valid = ds_train.split(0.7)
ds_valid, ds_test = ds_valid.split(0.7)
#####################################
#Define Model
#####################################
# create sigmoid hidden layer with 20 nodes, init weights in range -0.05 to 0.05 and add
# a bias with value 1
hidden_layer = mlp.Sigmoid(layer_name='h0', dim=1, irange=.05, init_bias=1.)
# softmax output layer
output_layer = mlp.Softmax(2, 'softmax', irange=.05)
layers = [hidden_layer, output_layer]
# create neural net
ann = mlp.MLP(layers, nvis=ds_train.nr_inputs)
#####################################
#Define Training
#####################################
#L1 Weight Decay
L1_cost = PL.costs.cost.SumOfCosts([PL.costs.cost.MethodCost(method='cost_from_X'), PL.costs.mlp.L1WeightDecay(coeffs=[0.1, 0.01])])
# momentum
def init_train():
# Initialize train object.
idpath = os.path.splitext(
os.path.abspath(__file__))[0] # ID for output files.
save_path = idpath + '.pkl'
# Dataset
#seed = 42
nvis = 32
dataset_train = pylearn2.datasets.improver2013.IMPROVER2013(
which_set='A_human_phospho_from_rat_phospho_replicate')
# Parameters
momentum_saturate = 500
# Model
model = pylearn2.models.mlp.MLP(layers=[
mlp.Sigmoid(layer_name='h0', dim=1000, istdev=.01),
mlp.Sigmoid(layer_name='y', dim=32, istdev=.001)
],
nvis=nvis)
# Algorithm
algorithm = pylearn2.training_algorithms.sgd.SGD(
batch_size=1000, # If changed, change learning rate!
learning_rate=
.1, # In dropout paper=10 for gradient averaged over batch. Depends on batchsize.
init_momentum=.5,
monitoring_dataset={
'train': dataset_train,
},
termination_criterion=pylearn2.termination_criteria.EpochCounter(
max_epochs=6000),
#termination_criterion=pylearn2.termination_criteria.Or(criteria=[
# pylearn2.termination_criteria.MonitorBased(
# channel_name="valid_objective",
# prop_decrease=0.00001,
# N=40),
# pylearn2.termination_criteria.EpochCounter(
# max_epochs=momentum_saturate)
# ]),
#cost=pylearn2.costs.cost.SumOfCosts(
# costs=[pylearn2.costs.mlp.Default()
# ]
#),
#cost = pylearn2.costs.mlp.dropout.Dropout(
# input_include_probs={'h0':1., 'h1':1., 'h2':1., 'h3':1., 'y':0.5},
# input_scales={ 'h0': 1., 'h1':1., 'h2':1., 'h3':1., 'y':2.}),
cost=pylearn2.costs.mlp.addgauss.AddGauss(input_noise_stdev={
'h0': 0.2,
'y': 0.2
}),
update_callbacks=pylearn2.training_algorithms.sgd.ExponentialDecay(
decay_factor=
1.000004, # Decreases by this factor every batch. (1/(1.000001^8000)^100
min_lr=.000001))
# Extensions
extensions = [
#pylearn2.train_extensions.best_params.MonitorBasedSaveBest(channel_name='train_y_misclass',save_path=save_path)
pylearn2.training_algorithms.sgd.MomentumAdjustor(
start=0,
saturate=momentum_saturate,
final_momentum=.99 # Dropout=.5->.99 over 500 epochs.
)
]
# Train
train = pylearn2.train.Train(dataset=dataset_train,
model=model,
algorithm=algorithm,
extensions=extensions,
save_path=save_path,
save_freq=100)
return train
