def create_vae(x=None, batch=batch_size):
x = T.matrix('features') if x is None else x
x = x / 255.
encoder = MLP(
activations=[Rectifier(), Logistic()],
dims=[img_dim**2, hidden_dim, 2*latent_dim],
weights_init=IsotropicGaussian(std=0.01, mean=0),
biases_init=Constant(0.01),
name='encoder'
)
encoder.initialize()
z_param = encoder.apply(x)
z_mean, z_log_std = z_param[:,latent_dim:], z_param[:,:latent_dim]
z = Sampling(theano_seed=seed).apply([z_mean, z_log_std], batch=batch_size)
decoder = MLP(
activations=[Rectifier(), Logistic()],
dims=[latent_dim, hidden_dim, img_dim**2],
weights_init=IsotropicGaussian(std=0.01, mean=0),
biases_init=Constant(0.01),
name='decoder'
)
decoder.initialize()
x_reconstruct = decoder.apply(z)
cost = VAEloss().apply(x, x_reconstruct, z_mean, z_log_std)
cost.name = 'vae_cost'
return cost
def test_convolutional_sequence_tied_biases_pushed_if_explicitly_set():
cnn = ConvolutionalSequence(sum([[
Convolutional(filter_size=(1, 1), num_filters=1, tied_biases=True),
Rectifier()
] for _ in range(3)], []),
num_channels=1,
image_size=(1, 1),
tied_biases=False)
cnn.allocate()
assert [
not child.tied_biases for child in cnn.children
if isinstance(child, Convolutional)
]
cnn = ConvolutionalSequence(sum(
[[Convolutional(filter_size=(1, 1), num_filters=1),
Rectifier()] for _ in range(3)], []),
num_channels=1,
image_size=(1, 1),
tied_biases=True)
cnn.allocate()
assert [
child.tied_biases for child in cnn.children
if isinstance(child, Convolutional)
]
def create_model(self, x, y, input_dim, tol=10e-5):
# Create the output of the MLP
mlp = MLP(
[Rectifier(), Rectifier(), Logistic()], [input_dim, 100, 100, 1],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
mlp.initialize()
probs = mlp.apply(x)
y = y.dimshuffle(0, 'x')
# Create the if-else cost function
true_p = (T.sum(y * probs) + tol) * 1.0 / (T.sum(y) + tol)
true_n = (T.sum((1 - y) * (1 - probs)) + tol) * \
1.0 / (T.sum(1 - y) + tol)
#p = (T.sum(y) + tol) / (y.shape[0] + tol)
theta = (1 - self.p) / self.p
numerator = (1 + self.beta**2) * true_p
denominator = self.beta**2 + theta + true_p - theta * true_n
Fscore = numerator / denominator
cost = -1 * Fscore
cost.name = "cost"
return mlp, cost, probs
def build_mlp(features_cat, features_int, labels):
mlp_int = MLP(activations=[Rectifier(), Rectifier()],
dims=[19, 50, 50],
weights_init=IsotropicGaussian(),
biases_init=Constant(0),
name='mlp_interval')
mlp_int.initialize()
mlp_cat = MLP(activations=[Logistic()],
dims=[320, 50],
weights_init=IsotropicGaussian(),
biases_init=Constant(0),
name='mlp_categorical')
mlp_cat.initialize()
mlp = MLP(activations=[Rectifier(), None],
dims=[50, 50, 1],
weights_init=IsotropicGaussian(),
biases_init=Constant(0))
mlp.initialize()
gated = mlp_cat.apply(features_cat) * mlp_int.apply(features_int)
prediction = mlp.apply(gated)
cost = MAPECost().apply(prediction, labels)
cg = ComputationGraph(cost)
print cg.variables
cg_dropout1 = apply_dropout(cg, [VariableFilter(roles=[OUTPUT])(cg.variables)[1], VariableFilter(roles=[OUTPUT])(cg.variables)[3]], .2)
cost_dropout1 = cg_dropout1.outputs[0]
return cost_dropout1, cg_dropout1.parameters, cost
def create_lenet_5():
feature_maps = [6, 16]
mlp_hiddens = [120, 84]
conv_sizes = [5, 5]
pool_sizes = [2, 2]
image_size = (28, 28)
output_size = 10
# The above are from LeCun's paper. The blocks example had:
# feature_maps = [20, 50]
# mlp_hiddens = [500]
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations, 1, image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='valid',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
return convnet
def __init__(self, config, **kwargs):
super(Model, self).__init__(config, **kwargs)
self.dest_mlp = MLP(
activations=[Rectifier()
for _ in config.dim_hidden_dest] + [Softmax()],
dims=[config.dim_hidden[-1]] + config.dim_hidden_dest +
[config.dim_output_dest],
name='dest_mlp')
self.time_mlp = MLP(
activations=[Rectifier()
for _ in config.dim_hidden_time] + [Softmax()],
dims=[config.dim_hidden[-1]] + config.dim_hidden_time +
[config.dim_output_time],
name='time_mlp')
self.dest_classes = theano.shared(numpy.array(
config.dest_tgtcls, dtype=theano.config.floatX),
name='dest_classes')
self.time_classes = theano.shared(numpy.array(
config.time_tgtcls, dtype=theano.config.floatX),
name='time_classes')
self.inputs.append('input_time')
self.children.extend([self.dest_mlp, self.time_mlp])
def build_mlp(features_int, features_cat, labels, labels_mean):
inputs = tensor.concatenate([features_int, features_cat], axis=1)
mlp = MLP(activations=[Rectifier(),
Rectifier(),
Rectifier(), None],
dims=[337, 800, 1200, 1],
weights_init=IsotropicGaussian(),
biases_init=Constant(1))
mlp.initialize()
prediction = mlp.apply(inputs)
cost = MAPECost().apply(prediction, labels, labels_mean)
cg = ComputationGraph(cost)
#cg_dropout0 = apply_dropout(cg, [VariableFilter(roles=[INPUT])(cg.variables)[1]], .2)
cg_dropout1 = apply_dropout(cg, [
VariableFilter(roles=[OUTPUT])(cg.variables)[1],
VariableFilter(roles=[OUTPUT])(cg.variables)[3],
VariableFilter(roles=[OUTPUT])(cg.variables)[5]
], .2)
cost_dropout1 = cg_dropout1.outputs[0]
return cost_dropout1, cg_dropout1.parameters, cost #cost, cg.parameters, cost #
def build_mlp(features_car_cat, features_car_int, features_nocar_cat,
features_nocar_int, features_cp, features_hascar, means, labels):
features = tensor.concatenate([
features_hascar, means['cp'][features_cp[:, 0]],
means['dep'][features_cp[:, 1]]
],
axis=1)
mlp = MLP(activations=[Rectifier(), Rectifier(), None],
dims=[5, 50, 50, 1],
weights_init=IsotropicGaussian(.1),
biases_init=Constant(0),
name='mlp')
mlp.initialize()
prediction = mlp.apply(features)
cost = MAPECost().apply(labels, prediction)
cg = ComputationGraph(cost)
input_var = VariableFilter(roles=[INPUT])(cg.variables)
print input_var
cg_dropout1 = apply_dropout(cg, [input_var[3], input_var[5]], .4)
cost_dropout1 = cg_dropout1.outputs[0]
return prediction, cost_dropout1, cg_dropout1.parameters, cost
def test_defaults_sequence2():
seq = DefaultsSequence(input_dim=(3, 4, 4),
lists=[
Convolutional(num_filters=10,
stride=(2, 2),
filter_size=(3, 3)),
BatchNormalization(),
Rectifier(),
Flattener(),
Linear(output_dim=10),
BatchNormalization(),
Rectifier(),
Linear(output_dim=12),
BatchNormalization(),
Rectifier()
])
seq.weights_init = Constant(1.0)
seq.biases_init = Constant(0.0)
seq.push_allocation_config()
seq.push_initialization_config()
seq.initialize()
x = T.tensor4('input')
y = seq.apply(x)
func_ = theano.function([x], [y])
x_val = np.ones((1, 3, 4, 4), dtype=theano.config.floatX)
res = func_(x_val)[0]
assert_allclose(res.shape, (1, 12))
def setup_ff_network(in_dim, out_dim, num_layers, num_neurons):
"""Setup a feedforward neural network.
Parameters
----------
in_dim : int
input dimension of network
out_dim : int
output dimension of network
num_layers : int
number of hidden layers
num_neurons : int
number of neurons of each layer
Returns
-------
net : object
network structure
"""
activations = [Rectifier()]
dims = [in_dim]
for i in xrange(num_layers):
activations.append(Rectifier())
dims.append(num_neurons)
dims.append(out_dim)
net = MLP(activations=activations,
dims=dims,
weights_init=IsotropicGaussian(),
biases_init=Constant(0.01))
return net
def __init__(self, n_out, dwin, vector_size, n_hidden_layer, **kwargs):
super(ConvPoolNlp, self).__init__(**kwargs)
self.vector_size = vector_size
self.n_hidden_layer = n_hidden_layer
self.dwin = dwin
self.n_out = n_out
self.rectifier = Rectifier()
"""
self.convolution = Convolutional(filter_size=(1,self.filter_size),num_filters=self.num_filter,num_channels=1,
weights_init=IsotropicGaussian(0.01), use_bias=False)
"""
# second dimension is of fixed size sum(vect_size) less the fiter_size borders
self.mlp = MLP(activations=[Rectifier()] * len(self.n_hidden_layer) +
[Identity()],
dims=[self.n_out] + self.n_hidden_layer + [2],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0.))
self.parameters = []
self.children = []
#self.children.append(self.lookup)
#self.children.append(self.convolution)
self.children.append(self.mlp)
self.children.append(self.rectifier)
def __init__(self, dim, mini_dim, summary_dim, **kwargs):
super(RNNwMini, self).__init__(**kwargs)
self.dim = dim
self.mini_dim = mini_dim
self.summary_dim = summary_dim
self.recurrent_layer = SimpleRecurrent(
dim=self.summary_dim,
activation=Rectifier(),
name='recurrent_layer',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
self.mini_recurrent_layer = SimpleRecurrent(
dim=self.mini_dim,
activation=Rectifier(),
name='mini_recurrent_layer',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
self.mini_to_main = Linear(self.dim + self.mini_dim,
self.summary_dim,
name='mini_to_main',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
self.children = [
self.recurrent_layer, self.mini_recurrent_layer, self.mini_to_main
]
def __init__(self, image_dimension, **kwargs):
layers = []
#############################################
# a first block with 2 convolutions of 32 (3, 3) filters
layers.append(Convolutional((3, 3), 32, border_mode='half'))
layers.append(Rectifier())
layers.append(Convolutional((3, 3), 32, border_mode='half'))
layers.append(Rectifier())
# maxpool with size=(2, 2)
layers.append(MaxPooling((2, 2)))
#############################################
# a 2nd block with 3 convolutions of 64 (3, 3) filters
layers.append(Convolutional((3, 3), 64, border_mode='half'))
layers.append(Rectifier())
layers.append(Convolutional((3, 3), 64, border_mode='half'))
layers.append(Rectifier())
layers.append(Convolutional((3, 3), 64, border_mode='half'))
layers.append(Rectifier())
# maxpool with size=(2, 2)
layers.append(MaxPooling((2, 2)))
#############################################
# a 3rd block with 4 convolutions of 128 (3, 3) filters
layers.append(Convolutional((3, 3), 128, border_mode='half'))
layers.append(Rectifier())
layers.append(Convolutional((3, 3), 128, border_mode='half'))
layers.append(Rectifier())
layers.append(Convolutional((3, 3), 128, border_mode='half'))
layers.append(Rectifier())
layers.append(Convolutional((3, 3), 128, border_mode='half'))
layers.append(Rectifier())
# maxpool with size=(2, 2)
layers.append(MaxPooling((2, 2)))
self.conv_sequence = ConvolutionalSequence(layers,
3,
image_size=image_dimension)
flattener = Flattener()
self.top_mlp = MLP(activations=[Rectifier(), Logistic()],
dims=[500, 1])
application_methods = [
self.conv_sequence.apply, flattener.apply, self.top_mlp.apply
]
super(VGGNet, self).__init__(application_methods,
biases_init=Constant(0),
weights_init=Uniform(width=.1),
**kwargs)
def test_fully_layer():
batch_size=2
x = T.tensor4();
y = T.ivector()
V = 200
layer_conv = Convolutional(filter_size=(5,5),num_filters=V,
name="toto",
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0.0))
# try with no bias
activation = Rectifier()
pool = MaxPooling(pooling_size=(2,2))
convnet = ConvolutionalSequence([layer_conv, activation, pool], num_channels=15,
image_size=(10,10),
name="conv_section")
convnet.push_allocation_config()
convnet.initialize()
output=convnet.apply(x)
batch_size=output.shape[0]
output_dim=np.prod(convnet.get_dim('output'))
result_conv = output.reshape((batch_size, output_dim))
mlp=MLP(activations=[Rectifier().apply], dims=[output_dim, 10],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0.0))
mlp.initialize()
output=mlp.apply(result_conv)
cost = T.mean(Softmax().categorical_cross_entropy(y.flatten(), output))
cg = ComputationGraph(cost)
W = VariableFilter(roles=[WEIGHT])(cg.variables)
B = VariableFilter(roles=[BIAS])(cg.variables)
W = W[0]; b = B[0]
inputs_fully = VariableFilter(roles=[INPUT], bricks=[Linear])(cg)
outputs_fully = VariableFilter(roles=[OUTPUT], bricks=[Linear])(cg)
var_input=inputs_fully[0]
var_output=outputs_fully[0]
[d_W,d_S,d_b] = T.grad(cost, [W, var_output, b])
d_b = d_b.dimshuffle(('x',0))
d_p = T.concatenate([d_W, d_b], axis=0)
x_value = 1e3*np.random.ranf((2,15, 10, 10))
f = theano.function([x,y], [var_input, d_S, d_p], allow_input_downcast=True, on_unused_input='ignore')
A, B, C= f(x_value, [5, 0])
A = np.concatenate([A, np.ones((2,1))], axis=1)
print 'A', A.shape
print 'B', B.shape
print 'C', C.shape
print lin.norm(C - np.dot(np.transpose(A), B), 'fro')
return
"""
def __init__(self,
filter_size,
num_filters,
num_channels,
batch_size=None,
mid_noise=False,
out_noise=False,
tied_noise=False,
tied_sigma=False,
noise_rate=None,
noise_batch_size=None,
prior_noise_level=None,
image_size=(None, None),
step=(1, 1),
**kwargs):
self.filter_size = filter_size
self.num_filters = num_filters
self.batch_size = batch_size
self.num_channels = num_channels
self.image_size = image_size
self.mid_noise = mid_noise
self.noise_batch_size = noise_batch_size
self.noise_rate = noise_rate
self.step = step
self.border_mode = 'half'
self.tied_biases = True
depth = 2
self.b0 = SpatialBatchNormalization(name='b0')
self.r0 = Rectifier(name='r0')
self.n0 = (SpatialNoise(name='n0',
noise_rate=self.noise_rate,
tied_noise=tied_noise,
tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level)
if mid_noise else None)
self.c0 = Convolutional(name='c0')
self.b1 = SpatialBatchNormalization(name='b1')
self.r1 = Rectifier(name='r1')
self.n1 = (SpatialNoise(name='n1',
noise_rate=self.noise_rate,
tied_noise=tied_noise,
tied_sigma=tied_sigma,
prior_noise_level=prior_noise_level)
if out_noise else None)
self.c1 = Convolutional(name='c1')
kwargs.setdefault('children', []).extend([
c for c in [
self.c0, self.b0, self.r0, self.n0, self.c1, self.b1, self.r1,
self.n1
] if c is not None
])
super(ResidualConvolutional, self).__init__(**kwargs)
def build_mlp(features_car_cat, features_car_int, features_nocar_cat,
features_nocar_int, features_cp, features_hascar, means, labels):
mlp_car = MLP(activations=[Rectifier(), Rectifier(), None],
dims=[8 + 185, 200, 200, 1],
weights_init=IsotropicGaussian(.1),
biases_init=Constant(0),
name='mlp_interval_car')
mlp_car.initialize()
mlp_nocar = MLP(activations=[Rectifier(), Rectifier(), None],
dims=[5 + 135, 200, 200, 1],
weights_init=IsotropicGaussian(.1),
biases_init=Constant(0),
name='mlp_interval_nocar')
mlp_nocar.initialize()
feature_car = tensor.concatenate((features_car_cat, features_car_int),
axis=1)
feature_nocar = tensor.concatenate(
(features_nocar_cat, features_nocar_int), axis=1)
prediction = mlp_nocar.apply(feature_nocar)
# gating with the last feature : does the dude own a car
prediction += tensor.addbroadcast(features_hascar,
1) * mlp_car.apply(feature_car)
prediction_loc, _, _, _, = \
build_mlp_onlyloc(features_car_cat, features_car_int,
features_nocar_cat, features_nocar_int,
features_cp, features_hascar,
means, labels)
prediction += prediction_loc
# add crm
mlp_crm = MLP(activations=[None],
dims=[1, 1],
weights_init=IsotropicGaussian(.1),
biases_init=Constant(0),
name='mlp_crm')
mlp_crm.initialize()
crm = features_nocar_int[:, 0][:, None]
prediction = prediction * mlp_crm.apply(crm)
cost = MAPECost().apply(labels, prediction)
cg = ComputationGraph(cost)
input_var = VariableFilter(roles=[INPUT])(cg.variables)
print input_var
cg_dropout1 = apply_dropout(cg, [input_var[6], input_var[7]], .4)
cost_dropout1 = cg_dropout1.outputs[0]
return prediction, cost_dropout1, cg_dropout1.parameters, cost
def apply_cnn(self, l_emb1, l_size1, l_emb2, l_size2, r_emb1, r_size1,
r_emb2, r_size2, embedding_size, mycnf):
assert l_size1 == r_size1
assert l_size2 == r_size2
assert l_size1 == l_size1
max_len = l_size1
fv_len = 0
filter_sizes = mycnf['cnn_config']['filter_sizes']
num_filters = mycnf['cnn_config']['num_filters']
for i, fw in enumerate(filter_sizes):
conv_left = ConvolutionalActivation(
activation=Rectifier().apply,
filter_size=(fw, embedding_size),
num_filters=num_filters,
num_channels=1,
image_size=(max_len, embedding_size),
name="conv" + str(fw) + l_emb1.name,
seed=self.curSeed)
conv_right = ConvolutionalActivation(
activation=Rectifier().apply,
filter_size=(fw, embedding_size),
num_filters=num_filters,
num_channels=1,
image_size=(max_len, embedding_size),
name="conv" + str(fw) + r_emb1.name,
seed=self.curSeed)
pooling = MaxPooling((max_len - fw + 1, 1), name="pool" + str(fw))
initialize([conv_left, conv_right])
l_convinp1 = l_emb1.flatten().reshape(
(l_emb1.shape[0], 1, max_len, embedding_size))
l_convinp2 = l_emb2.flatten().reshape(
(l_emb2.shape[0], 1, max_len, embedding_size))
l_pool1 = pooling.apply(conv_left.apply(l_convinp1)).flatten(2)
l_pool2 = pooling.apply(conv_left.apply(l_convinp2)).flatten(2)
r_convinp1 = r_emb1.flatten().reshape(
(r_emb1.shape[0], 1, max_len, embedding_size))
r_convinp2 = r_emb2.flatten().reshape(
(r_emb2.shape[0], 1, max_len, embedding_size))
r_pool1 = pooling.apply(conv_right.apply(r_convinp1)).flatten(2)
r_pool2 = pooling.apply(conv_right.apply(r_convinp2)).flatten(2)
onepools1 = T.concatenate([l_pool1, r_pool1], axis=1)
onepools2 = T.concatenate([l_pool2, r_pool2], axis=1)
fv_len += conv_left.num_filters * 2
if i == 0:
outpools1 = onepools1
outpools2 = onepools2
else:
outpools1 = T.concatenate([outpools1, onepools1], axis=1)
outpools2 = T.concatenate([outpools2, onepools2], axis=1)
return outpools1, outpools2, fv_len
def generate_elementary_block3(self, index, to_index):
number_of_channels = 512
name_conv_0 = 'fconv7_' + str(index)
name_relu_0 = 'relu7_' + str(index)
name_conv_1 = 'fconv7_' + str(index) + 'to' + str(to_index) + '_step1'
name_relu_1 = 'relu7_' + str(index) + 'to' + str(to_index) + '_step1'
name_conv_2 = 'fconv7_' + str(index) + 'to' + str(to_index) + '_step2'
name_conv_3 = 'fconv7_output_' + str(index)
return [Convolutional(filter_size=(1,1), num_filters = 128, border_mode = (0,0), use_bias=True, tied_biases=True, name=name_conv_0, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = number_of_channels), \
ParallelSum3(), Rectifier(name=name_relu_0), \
Convolutional(filter_size=(7,7), num_filters = 64, border_mode = (3,3), use_bias=True, tied_biases=True, name=name_conv_1, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 128), \
Rectifier(name=name_relu_1), \
Convolutional(filter_size=(7,7), num_filters = 128, border_mode = (3,3), use_bias=True, tied_biases=True, name=name_conv_2, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 64), \
Convolutional(filter_size=(1,1), num_filters = 1, border_mode = (0,0), use_bias=True, tied_biases=True, name=name_conv_3, biases_init=Constant(0.), weights_init=IsotropicGaussian(0.01), num_channels = 128)]
def test_convolutional_layer():
x = tensor.tensor4('x')
num_channels = 4
batch_size = 5
pooling_size = 3
num_filters = 3
filter_size = (3, 3)
activation = Rectifier().apply
conv = ConvolutionalLayer(activation,
filter_size,
num_filters, (pooling_size, pooling_size),
num_channels,
image_size=(17, 13),
weights_init=Constant(1.),
biases_init=Constant(5.))
conv.initialize()
y = conv.apply(x)
func = function([x], y)
x_val = numpy.ones((batch_size, num_channels, 17, 13),
dtype=theano.config.floatX)
assert_allclose(
func(x_val),
numpy.prod(filter_size) * num_channels * numpy.ones(
(batch_size, num_filters, 5, 4)) + 5)
def __init__(self,
input_dim,
dim,
mlp_hidden_dims,
batch_size,
image_shape,
patch_shape,
activation=None,
**kwargs):
super(LSTMAttention, self).__init__(**kwargs)
self.dim = dim
self.image_shape = image_shape
self.patch_shape = patch_shape
self.batch_size = batch_size
non_lins = [Rectifier()] * (len(mlp_hidden_dims) - 1) + [None]
mlp_dims = [input_dim + dim] + mlp_hidden_dims
mlp = MLP(non_lins,
mlp_dims,
weights_init=self.weights_init,
biases_init=self.biases_init,
name=self.name + '_mlp')
hyperparameters = {}
hyperparameters["cutoff"] = 3
hyperparameters["batched_window"] = True
cropper = LocallySoftRectangularCropper(
patch_shape=patch_shape,
hyperparameters=hyperparameters,
kernel=Gaussian())
if not activation:
activation = Tanh()
self.children = [activation, mlp, cropper]
def create_cnn_general(embedded_x, mycnf, max_len, embedding_size, inp_conv=False):
fv_len = 0
filter_sizes = mycnf['cnn_config']['filter_sizes']
num_filters = mycnf['cnn_config']['num_filters']
for i, fw in enumerate(filter_sizes):
conv = ConvolutionalActivation(
activation=Rectifier().apply,
filter_size=(fw, embedding_size),
num_filters=num_filters,
num_channels=1,
image_size=(max_len, embedding_size),
name="conv"+str(fw)+embedded_x.name)
pooling = MaxPooling((max_len-fw+1, 1), name="pool"+str(fw)+embedded_x.name)
initialize([conv])
if inp_conv:
convinp = embedded_x
else:
convinp = embedded_x.flatten().reshape((embedded_x.shape[0], 1, max_len, embedding_size))
onepool = pooling.apply(conv.apply(convinp)).flatten(2)
if i == 0:
outpools = onepool
else:
outpools = T.concatenate([outpools, onepool], axis=1)
fv_len += conv.num_filters
return outpools, fv_len
def construct_mlp(name,
hidden_dims,
input_dim,
initargs,
batch_normalize,
activations=None):
if not hidden_dims:
return FeedforwardIdentity(dim=input_dim)
if not activations:
activations = [Rectifier() for dim in hidden_dims]
elif not isinstance(activations, collections.Iterable):
activations = [activations] * len(hidden_dims)
assert len(activations) == len(hidden_dims)
dims = [input_dim] + hidden_dims
wrapped_activations = [
NormalizedActivation(shape=[hidden_dim],
name="activation_%i" % i,
batch_normalize=batch_normalize,
activation=activation)
for i, (hidden_dim,
activation) in enumerate(zip(hidden_dims, activations))
]
mlp = MLP(name=name,
activations=wrapped_activations,
dims=dims,
**initargs)
# biases are handled by our activation function
for layer in mlp.linear_transformations:
layer.use_bias = False
return mlp
def test_convolutional_sequence():
x = tensor.tensor4('x')
num_channels = 4
pooling_size = 3
batch_size = 5
activation = Rectifier().apply
conv = ConvolutionalLayer(activation, (3, 3),
5, (pooling_size, pooling_size),
weights_init=Constant(1.),
biases_init=Constant(5.))
conv2 = ConvolutionalActivation(activation, (2, 2),
4,
weights_init=Constant(1.))
seq = ConvolutionalSequence([conv, conv2],
num_channels,
image_size=(17, 13))
seq.push_allocation_config()
assert conv.num_channels == 4
assert conv2.num_channels == 5
conv2.convolution.use_bias = False
y = seq.apply(x)
seq.initialize()
func = function([x], y)
x_val = numpy.ones((batch_size, 4, 17, 13), dtype=theano.config.floatX)
y_val = (numpy.ones((batch_size, 4, 4, 3)) * (9 * 4 + 5) * 4 * 5)
assert_allclose(func(x_val), y_val)
def test_convolutional_sequence_with_no_input_size():
# suppose x is outputted by some RNN
x = tensor.tensor4('x')
filter_size = (1, 1)
num_filters = 2
num_channels = 1
pooling_size = (1, 1)
conv = Convolutional(filter_size,
num_filters,
tied_biases=False,
weights_init=Constant(1.),
biases_init=Constant(1.))
act = Rectifier()
pool = MaxPooling(pooling_size)
bad_seq = ConvolutionalSequence([conv, act, pool],
num_channels,
tied_biases=False)
assert_raises_regexp(ValueError, 'Cannot infer bias size \S+',
bad_seq.initialize)
seq = ConvolutionalSequence([conv, act, pool],
num_channels,
tied_biases=True)
try:
seq.initialize()
out = seq.apply(x)
except TypeError:
assert False, "This should have succeeded"
assert out.ndim == 4
def create_OLD_kim_cnn(layer0_input, embedding_size, input_len, config, pref):
'''
One layer convolution with the same filtersize
'''
filter_width_list = [
int(fw) for fw in config[pref + '_filterwidth'].split()
]
print filter_width_list
num_filters = int(config[pref + '_num_filters'])
totfilters = 0
for i, fw in enumerate(filter_width_list):
num_feature_map = input_len - fw + 1 #39
conv = Convolutional(filter_size=(fw, embedding_size),
num_filters=num_filters,
num_channels=1,
image_size=(input_len, embedding_size),
name="conv" + str(fw))
pooling = MaxPooling((num_feature_map, 1), name="pool" + str(fw))
initialize([conv])
totfilters += num_filters
outpool = Flattener(name="flat" + str(fw)).apply(
Rectifier(name=pref + 'act_' + str(fw)).apply(
pooling.apply(conv.apply(layer0_input))))
if i == 0:
outpools = outpool
else:
outpools = T.concatenate([outpools, outpool], axis=1)
name_rep_len = totfilters
return outpools, name_rep_len
def __init__(self,
filter_size,
num_filters,
num_channels,
noise_batch_size,
image_size=(None, None),
step=(1, 1),
border_mode='valid',
tied_biases=True,
prior_mean=0,
prior_noise_level=0,
**kwargs):
self.convolution = Convolutional()
self.rectifier = Rectifier()
self.mask = Convolutional(name='mask')
children = [self.convolution, self.rectifier, self.mask]
kwargs.setdefault('children', []).extend(children)
super(NoisyConvolutional2, self).__init__(**kwargs)
self.filter_size = filter_size
self.num_filters = num_filters
self.num_channels = num_channels
self.noise_batch_size = noise_batch_size
self.image_size = image_size
self.step = step
self.border_mode = border_mode
self.tied_biases = tied_biases
self.prior_mean = prior_mean
self.prior_noise_level = prior_noise_level
def create_kim_cnn(layer0_input, embedding_size, input_len, config, pref):
'''
One layer convolution with different filter-sizes and maxpooling
'''
filter_width_list = [
int(fw) for fw in config[pref + '_filterwidth'].split()
]
print filter_width_list
num_filters = int(config[pref + '_num_filters'])
#num_filters /= len(filter_width_list)
totfilters = 0
for i, fw in enumerate(filter_width_list):
num_feature_map = input_len - fw + 1 #39
conv = Convolutional(image_size=(input_len, embedding_size),
filter_size=(fw, embedding_size),
num_filters=min(int(config[pref + '_maxfilter']),
num_filters * fw),
num_channels=1)
totfilters += conv.num_filters
initialize2(conv, num_feature_map)
conv.name = pref + 'conv_' + str(fw)
convout = conv.apply(layer0_input)
pool_layer = MaxPooling(pooling_size=(num_feature_map, 1))
pool_layer.name = pref + 'pool_' + str(fw)
act = Rectifier()
act.name = pref + 'act_' + str(fw)
outpool = act.apply(pool_layer.apply(convout)).flatten(2)
if i == 0:
outpools = outpool
else:
outpools = T.concatenate([outpools, outpool], axis=1)
name_rep_len = totfilters
return outpools, name_rep_len
def __init__(self,
image_shape=None,
output_size=None,
noise_batch_size=None,
noise_without_rectifier=False,
noise_after_rectifier=False,
**kwargs):
self.num_channels = 3
self.image_shape = image_shape or (32, 32)
self.output_size = output_size or 10
self.noise_batch_size = noise_batch_size
conv_parameters = [(96, 3, 1, 'half', Convolutional),
(96, 3, 1, 'half', Convolutional),
(96, 3, 2, 'half', NoisyConvolutional),
(192, 3, 1, 'half', Convolutional),
(192, 3, 1, 'half', Convolutional),
(192, 3, 2, 'half', NoisyConvolutional),
(192, 3, 1, 'half', Convolutional),
(192, 1, 1, 'valid', Convolutional),
(10, 1, 1, 'valid', Convolutional)]
fc_layer = 10
self.convolutions = []
layers = []
for i, (num_filters, filter_size, conv_step, border_mode,
cls) in enumerate(conv_parameters):
if cls == NoisyConvolutional and noise_after_rectifier:
cls = NoisyConvolutional2
layer = cls(filter_size=(filter_size, filter_size),
num_filters=num_filters,
step=(conv_step, conv_step),
border_mode=border_mode,
tied_biases=True,
name='conv_{}'.format(i))
if cls == NoisyConvolutional or cls == NoisyConvolutional2:
layer.noise_batch_size = self.noise_batch_size
self.convolutions.append(layer)
layers.append(layer)
if cls != NoisyConvolutional2 and not noise_without_rectifier:
layers.append(Rectifier())
self.conv_sequence = ConvolutionalSequence(layers,
self.num_channels,
image_size=self.image_shape)
# The AllConvNet applies average pooling to combine top-level
# features across the image.
self.flattener = GlobalAverageFlattener()
# Then it inserts one final 10-way FC layer before softmax
# self.top_mlp = MLP([Rectifier(), Softmax()],
# [conv_parameters[-1][0], fc_layer, self.output_size])
self.top_softmax = Softmax()
application_methods = [
self.conv_sequence.apply, self.flattener.apply,
self.top_softmax.apply
]
super(NoisyAllConvNet, self).__init__(application_methods, **kwargs)
def generation(z_list, n_latent, hu_decoder, n_out, y):
logger.info('in generation: n_latent: %d, hu_decoder: %d', n_latent,
hu_decoder)
if hu_decoder == 0:
return generation_simple(z_list, n_latent, n_out, y)
mlp1 = MLP(activations=[Rectifier()],
dims=[n_latent, hu_decoder],
name='latent_to_hidDecoder')
initialize([mlp1])
hid_to_out = Linear(name='hidDecoder_to_output',
input_dim=hu_decoder,
output_dim=n_out)
initialize([hid_to_out])
mysigmoid = Logistic(name='y_hat_vae')
agg_logpy_xz = 0.
agg_y_hat = 0.
for i, z in enumerate(z_list):
y_hat = mysigmoid.apply(hid_to_out.apply(
mlp1.apply(z))) #reconstructed x
agg_logpy_xz += cross_entropy_loss(y_hat, y)
agg_y_hat += y_hat
agg_logpy_xz /= len(z_list)
agg_y_hat /= len(z_list)
return agg_y_hat, agg_logpy_xz
def build_model(images, labels):
# Construct a bottom convolutional sequence
bottom_conv_sequence = convolutional_sequence((3,3), 16, (160, 160))
bottom_conv_sequence._push_allocation_config()
# Flatten layer
flattener = Flattener()
# Construct a top MLP
conv_out_dim = numpy.prod(bottom_conv_sequence.get_dim('output'))
#top_mlp = MLP([Rectifier(name='non_linear_9'), Softmax(name='non_linear_11')], [conv_out_dim, 1024, 10], weights_init=IsotropicGaussian(), biases_init=Constant(0))
top_mlp = BatchNormalizedMLP([Rectifier(name='non_linear_9'), Softmax(name='non_linear_11')], [conv_out_dim, 1024, 10], weights_init=IsotropicGaussian(), biases_init=Constant(0))
# Construct feedforward sequence
ss_seq = FeedforwardSequence([bottom_conv_sequence.apply, flattener.apply, top_mlp.apply])
ss_seq.push_initialization_config()
ss_seq.initialize()
prediction = ss_seq.apply(images)
cost_noreg = CategoricalCrossEntropy().apply(labels.flatten(), prediction)
# add regularization
selector = Selector([top_mlp])
Ws = selector.get_parameters('W')
mlp_brick_name = 'batchnormalizedmlp'
W0 = Ws['/%s/linear_0.W' % mlp_brick_name]
W1 = Ws['/%s/linear_1.W' % mlp_brick_name]
cost = cost_noreg + .01 * (W0 ** 2).mean() + .01 * (W1 ** 2).mean()
return cost
