def get_init(features, initializer_class, nhid=5):
initializer = initializer_class.build({}, graph=features.graph)
if hasattr(initializer, "Input"):
initializer.Input.resize(2)
initializer.Input[0].connect(features.Output)
nvis = features.Output.meta.shape[1]
layer = Sigmoid(layer_name="a", irange=0, dim=nhid)
# layer needs to be initialized by MLP first
MLP(layers=[layer], nvis=nvis)
initializer.init_layer(layer, nvis=nvis, nhid=nhid)
weights = layer.get_weights()
biases = layer.get_biases()
return weights, biases
def test_kl():
"""
Test whether function kl() has properly processed the input.
"""
init_mode = theano.config.compute_test_value
theano.config.compute_test_value = 'raise'
try:
mlp = MLP(layers=[Sigmoid(dim=10, layer_name='Y', irange=0.1)],
nvis=10)
X = mlp.get_input_space().make_theano_batch()
Y = mlp.get_output_space().make_theano_batch()
X.tag.test_value = np.random.random(
get_debug_values(X)[0].shape).astype(theano.config.floatX)
Y_hat = mlp.fprop(X)
# This call should not raise any error:
ave = kl(Y, Y_hat, 1)
# The following calls should raise ValueError exceptions:
Y.tag.test_value[2][3] = 1.1
np.testing.assert_raises(ValueError, kl, Y, Y_hat, 1)
Y.tag.test_value[2][3] = -0.1
np.testing.assert_raises(ValueError, kl, Y, Y_hat, 1)
finally:
theano.config.compute_test_value = init_mode
def test_convnet():
layers = []
dataset = get_dataset()
input_space = Conv2DSpace(shape=[256, 256], num_channels=1)
conv_layer = ConvRectifiedLinear(output_channels=12,
irange=.005,
layer_name="h0",
kernel_shape=[88, 88],
kernel_stride=[8, 8],
pool_shape=[1, 1],
pool_stride=[1, 1],
max_kernel_norm=1.932)
layers.append(conv_layer)
maxout_layer = Maxout(layer_name="h1",
irange=.005,
num_units=600,
num_pieces=4,
max_col_norm=1.932)
layers.append(maxout_layer)
sigmoid_layer = Sigmoid(layer_name="y",
dim=484,
monitor_style="detection",
irange=.005)
layers.append(sigmoid_layer)
model = MLP(batch_size=100, layers=layers, input_space=input_space)
trainer = get_layer_trainer_sgd(model, dataset)
trainer.main_loop()
def testLSF(self):
init = LeastSquaresWeightInitializer.build({}, graph=Graph())
init.Input.resize(2)
init.Input[0].setValue(self.X)
init.Input[1].setValue(self.y)
for k in (2, 3, 5, 7):
layer = Sigmoid(layer_name="a", irange=0, dim=k)
# layer needs to be initialized by MLP first
MLP(layers=[layer], nvis=self.d)
np.random.seed(123)
init.init_layer(layer, nvis=self.d, nhid=k)
weights = layer.get_weights()
np.testing.assert_array_equal(weights.shape, (self.d, k))
assert np.abs(weights).sum() > 0
def get_layer_sigmoid(self, layer_id, layer_name):
row = self.db.executeSQL(
"""
SELECT dim,irange,istdev,sparse_init,sparse_stdev,include_prob,init_bias,
W_lr_scale,b_lr_scale,max_col_norm,max_row_norm
FROM hps3.layer_sigmoid
WHERE layer_id = %s
""", (layer_id, ), self.db.FETCH_ONE)
if not row or row is None:
raise HPSData("No sigmoid layer for layer_id=" \
+str(layer_id))
(dim, irange, istdev, sparse_init, sparse_stdev, include_prob,
init_bias, W_lr_scale, b_lr_scale, max_col_norm, max_row_norm) = row
return Sigmoid(layer_name=layer_name,
dim=dim,
irange=irange,
istdev=istdev,
sparse_init=sparse_init,
sparse_stdev=sparse_stdev,
include_prob=include_prob,
init_bias=init_bias,
W_lr_scale=W_lr_scale,
b_lr_scale=b_lr_scale,
max_col_norm=max_col_norm,
max_row_norm=max_row_norm)
def get_layer_sigmoid(self, layer):
return Sigmoid(layer_name=layer.layer_name,dim=layer.dim,irange=layer.irange,
istdev=layer.istdev,sparse_init=layer.sparse_init,
sparse_stdev=layer.sparse_stdev, include_prob=layer.include_prob,
init_bias=layer.init_bias,W_lr_scale=layer.W_lr_scale,
b_lr_scale=layer.b_lr_scale,max_col_norm=layer.max_col_norm,
max_row_norm=layer.max_row_norm)
def get_Sigmoid_Layer(self, structure, i=0):
n_input, n_output = structure
config = {
'dim': n_output,
'layer_name': ("s%d" % i),
'irange': 0.05,
}
return Sigmoid(**config)
def test_sigmoid_detection_cost():
# This is only a smoke test: verifies that it compiles and runs,
# not any particular value.
rng = np.random.RandomState(0)
y = (rng.uniform(size=(4, 3)) > 0.5).astype('uint8')
X = theano.shared(rng.uniform(size=(4, 2)))
model = MLP(nvis=2, layers=[Sigmoid(monitor_style='detection', dim=3,
layer_name='y', irange=0.8)])
y_hat = model.fprop(X)
model.cost(y, y_hat).eval()
def test_correctness():
"""
Test that the cost function works with float64
"""
x_train, y_train, x_valid, y_valid = create_dataset()
trainset = DenseDesignMatrix(X=np.array(x_train), y=y_train)
validset = DenseDesignMatrix(X=np.array(x_valid), y=y_valid)
n_inputs = trainset.X.shape[1]
n_outputs = 1
n_hidden = 10
hidden_istdev = 4 * (6 / float(n_inputs + n_hidden)) ** 0.5
output_istdev = 4 * (6 / float(n_hidden + n_outputs)) ** 0.5
model = MLP(layers=[Sigmoid(dim=n_hidden, layer_name='hidden',
istdev=hidden_istdev),
Sigmoid(dim=n_outputs, layer_name='output',
istdev=output_istdev)],
nvis=n_inputs, seed=[2013, 9, 16])
termination_criterion = And([EpochCounter(max_epochs=1),
MonitorBased(prop_decrease=1e-7,
N=2)])
cost = SumOfCosts([(0.99, Default()),
(0.01, L1WeightDecay({}))])
algo = SGD(1e-1,
update_callbacks=[ExponentialDecay(decay_factor=1.00001,
min_lr=1e-10)],
cost=cost,
monitoring_dataset=validset,
termination_criterion=termination_criterion,
monitor_iteration_mode='even_shuffled_sequential',
batch_size=2)
train = Train(model=model, dataset=trainset, algorithm=algo)
train.main_loop()
def testPCA(self):
init = PCAWeightInitializer.build({}, graph=Graph())
init.Input.resize(2)
init.Input[0].setValue(self.X)
init.Input[1].setValue(self.y)
for k in (2, 4, 6, 8):
layer = Sigmoid(layer_name="a", irange=0, dim=k)
# layer needs to be initialized by MLP first
MLP(layers=[layer], nvis=self.d)
np.random.seed(123)
init.init_layer(layer, nvis=self.d, nhid=k)
weights = layer.get_weights()
np.testing.assert_array_equal(weights.shape, (self.d, k))
assert np.abs(weights).sum() > 0
if k <= 2*self.d:
for i in range(k):
for j in range(i+2, k, 2):
dot = np.dot(weights[:, i], weights[:, j])
np.testing.assert_almost_equal(dot, 0, decimal=4)
def test_sigmoid_layer_misclass_reporting():
mlp = MLP(nvis=3, layers=[Sigmoid(layer_name='h0', dim=1, irange=0.005,
monitor_style='classification')])
target = theano.tensor.matrix(dtype=theano.config.floatX)
batch = theano.tensor.matrix(dtype=theano.config.floatX)
rval = mlp.layers[0].get_monitoring_channels_from_state(mlp.fprop(batch), target)
f = theano.function([batch, target], [tensor.gt(mlp.fprop(batch), 0.5),
rval['misclass']],
allow_input_downcast=True)
rng = np.random.RandomState(0)
for _ in range(10): # repeat a few times for statistical strength
targets = (rng.uniform(size=(30, 1)) > 0.5).astype('uint8')
out, misclass = f(rng.normal(size=(30, 3)), targets)
np.testing.assert_allclose((targets != out).mean(), misclass)
def generateNonConvRegressor(teacher_hintlayer, student_output_space):
dim = teacher_hintlayer.output_space.get_total_dimension()
layer_name = 'hint_regressor'
irng = 0.05
mcn = 0.9
if isinstance(teacher_hintlayer, MaxoutConvC01B):
hint_reg_layer = Maxout(layer_name,
dim,
teacher_hintlayer.num_pieces,
irange=irng,
max_col_norm=mcn)
elif isinstance(teacher_hintlayer, ConvRectifiedLinear):
hint_reg_layer = RectifiedLinear(dim=dim,
layer_name=layer_name,
irange=irng,
max_col_norm=mcn)
elif isinstance(teacher_hintlayer, ConvElemwise) or isinstance(
teacher_hintlayer, ConvElemwisePL2):
if isinstance(teacher_hintlayer.nonlinearity,
RectifierConvNonlinearity):
hint_reg_layer = RectifiedLinear(dim=dim,
layer_name=layer_name,
irange=irng,
max_col_norm=mcn)
elif isinstance(teacher_hintlayer.nonlinearity,
SigmoidConvNonlinearity):
hint_reg_layer = Sigmoid(dim=dim,
layer_name=layer_name,
irange=irng,
max_col_norm=mcn)
elif isinstance(teacher_hintlayer.nonlinearity, TanhConvNonlinearity):
hint_reg_layer = Tanh(dim=dim,
layer_name=layer_name,
irange=irng,
max_col_norm=mcn)
else:
raise AssertionError("Unknown layer type")
else:
raise AssertionError("Unknown fully-connected layer type")
return hint_reg_layer
def construct_dbn_from_stack(stack):
# some settings
irange = 0.05
layers = []
for ii, layer in enumerate(stack.layers()):
layers.append(
Sigmoid(dim=layer.nhid,
layer_name='h' + str(ii),
irange=irange,
max_col_norm=2.))
nc = 159 if SUBMODEL == 1 else 8
# softmax layer at then end for classification
layers.append(Softmax(n_classes=nc, layer_name='y', irange=irange))
dbn = MLP(layers=layers, nvis=stack.layers()[0].get_input_space().dim)
# copy weigths to DBN
for ii, layer in enumerate(stack.layers()):
dbn.layers[ii].set_weights(layer.get_weights())
dbn.layers[ii].set_biases(layer.hidbias.get_value(borrow=False))
return dbn
def get_convnet(img_shape=[256, 256],
output_channels=16,
kernel_shape=[88, 88],
kernel_stride=[8, 8]):
layers = []
dataset = get_dataset()
input_space = Conv2DSpace(shape=img_shape, num_channels=1)
conv_layer = ConvRectifiedLinear(output_channels=output_channels,
irange=.005,
layer_name="h0",
kernel_shape=kernel_shape,
kernel_stride=kernel_stride,
pool_shape=[1, 1],
pool_stride=[1, 1],
max_kernel_norm=1.932)
layers.append(conv_layer)
maxout_layer = Maxout(layer_name="h1",
irange=.005,
num_units=600,
num_pieces=4,
max_col_norm=1.932)
layers.append(maxout_layer)
conv_out_dim = ((img_shape[0] - kernel_shape[0]) / kernel_stride[0] + 1)**2
sigmoid_layer = Sigmoid(layer_name="y",
dim=conv_out_dim,
monitor_style="detection",
irange=.005)
layers.append(sigmoid_layer)
model = MLP(batch_size=100, layers=layers, input_space=input_space)
return model
from pylearn2.costs.cost import MethodCost
from pylearn2.datasets.mnist import MNIST
from pylearn2.models.mlp import MLP, Sigmoid, Softmax
from pylearn2.train import Train
from pylearn2.training_algorithms.sgd import SGD
from pylearn2.training_algorithms.learning_rule import Momentum, MomentumAdjustor
from pylearn2.termination_criteria import EpochCounter
train_set = MNIST(which_set='train', start=0, stop=50000)
valid_set = MNIST(which_set='train', start=50000, stop=60000)
test_set = MNIST(which_set='test')
model = MLP(nvis=784,
layers=[Sigmoid(layer_name='h', dim=500, irange=0.01),
Softmax(layer_name='y', n_classes=10, irange=0.01)])
algorithm = SGD(batch_size=100, learning_rate=0.01,
learning_rule=Momentum(init_momentum=0.5),
monitoring_dataset={'train': train_set,
'valid': valid_set,
'test': test_set},
cost=MethodCost('cost_from_X'),
termination_criterion=EpochCounter(10))
train = Train(dataset=train_set, model=model, algorithm=algorithm,
save_path="mnist_example.pkl", save_freq=1,
extensions=[MomentumAdjustor(start=5, saturate=6,
final_momentum=0.95)])
train.main_loop()
kernel_shape=[5, 5],
pool_shape=[2, 2],
pool_stride=[2, 2],
layer_name="h1",
irange=0.1,
border_mode="full")
h2 = ConvRectifiedLinear(output_channels=32,
kernel_shape=[5, 5],
pool_shape=[2, 2],
pool_stride=[2, 2],
layer_name="h2",
irange=0.1,
border_mode="full")
h3 = Sigmoid(layer_name="h3", dim=100, irange=0.01)
h4 = Sigmoid(layer_name="h4", dim=100, irange=0.01)
y = Softmax(n_classes=2, layer_name="y", irange=0.1)
inputSpace = Conv2DSpace(shape=[cropSize, cropSize], num_channels=3)
model = MLP(layers=[h0, h1, h2, h3, h4, y],
batch_size=batchSize,
input_space=inputSpace)
algorithm = SGD(learning_rate=1e-3,
cost=MethodCost("cost_from_X"),
batch_size=batchSize,
monitoring_batch_size=batchSize,
