def create_model(self):
input_dim = self.input_dim
x = self.x
x_to_h = Linear(input_dim,
input_dim * 4,
name='x_to_h',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
lstm = LSTM(input_dim,
name='lstm',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
h_to_o = Linear(input_dim,
1,
name='h_to_o',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
x_transform = x_to_h.apply(x)
self.x_to_h = x_to_h
self.lstm = lstm
self.h_to_o = h_to_o
h, c = lstm.apply(x_transform)
# only values of hidden units of the last timeframe are used for
# the classification
probs = h_to_o.apply(h[-1])
return probs
def _build_bricks(self, *args, **kwargs):
# Build lookup tables
self.word_embed = self._embed(len(self.dataset.word2index),
self.config.word_embed_dim,
name='word_embed')
self.user_embed = self._embed(len(self.dataset.user2index),
self.config.user_embed_dim,
name="user_embed")
self.hashtag_embed = self._embed(len(self.dataset.hashtag2index),
self.config.lstm_dim +
self.config.user_embed_dim,
name='hashtag_embed')
# Build text encoder
self.mlstm_ins = Linear(input_dim=self.config.word_embed_dim,
output_dim=4 * self.config.lstm_dim,
name='mlstm_in')
self.mlstm_ins.weights_init = IsotropicGaussian(
std=numpy.sqrt(2) /
numpy.sqrt(self.config.word_embed_dim + self.config.lstm_dim))
self.mlstm_ins.biases_init = Constant(0)
self.mlstm_ins.initialize()
self.mlstm = MLSTM(self.config.lstm_time,
self.config.lstm_dim,
shared=False)
self.mlstm.weights_init = IsotropicGaussian(
std=numpy.sqrt(2) /
numpy.sqrt(self.config.word_embed_dim + self.config.lstm_dim))
self.mlstm.biases_init = Constant(0)
self.mlstm.initialize()
def lllistool(i, inp, func):
if func == LSTM:
NUMS[i+1] *= 4
sdim = DIMS[i]
if func == SimpleRecurrent or func == LSTM:
sdim = DIMS[i] + DIMS[i+1]
l = Linear(input_dim=DIMS[i], output_dim=DIMS[i+1] * NUMS[i+1],
weights_init=IsotropicGaussian(std=sdim**(-0.5)),
biases_init=IsotropicGaussian(std=sdim**(-0.5)),
name='Lin{}'.format(i))
l.initialize()
if func == SimpleRecurrent:
gong = func(dim=DIMS[i+1], activation=Rectifier(), weights_init=IsotropicGaussian(std=sdim**(-0.5)))
gong.initialize()
ret = gong.apply(l.apply(inp))
elif func == LSTM:
gong = func(dim=DIMS[i+1], activation=Tanh(), weights_init=IsotropicGaussian(std=sdim**(-0.5)))
gong.initialize()
print(inp)
ret, _ = gong.apply(
l.apply(inp),
T.zeros((inp.shape[1], DIMS[i+1])),
T.zeros((inp.shape[1], DIMS[i+1])),
)
elif func == SequenceGenerator:
gong = func(
readout=None,
transition=SimpleRecurrent(dim=100, activation=Rectifier(), weights_init=IsotropicGaussian(std=0.1)))
ret = None
elif func == None:
ret = l.apply(inp)
else:
gong = func()
ret = gong.apply(l.apply(inp))
return ret
def create_base_model(self, x, y, input_dim, interim_dim=30):
# Create the output of the MLP
mlp = MLP([Tanh(), Tanh(), Tanh()], [input_dim, 60, 60, interim_dim],
weights_init=IsotropicGaussian(0.001),
biases_init=Constant(0))
mlp.initialize()
inter = mlp.apply(x)
fine_tuner = MLP([Logistic()], [interim_dim, 1],
weights_init=IsotropicGaussian(0.001),
biases_init=Constant(0))
fine_tuner.initialize()
probs = fine_tuner.apply(inter)
#sq_err = BinaryCrossEntropy()
err = T.sqr(y.flatten() - probs.flatten())
# cost = T.mean(err * y.flatten() * (1 - self.p) + err *
# (1 - y.flatten()) * self.p)
cost = T.mean(err)
#cost = sq_err.apply(probs.flatten(), y.flatten())
# cost = T.mean(y.flatten() * T.log(probs.flatten()) +
# (1 - y.flatten()) * T.log(1 - probs.flatten()))
cost.name = 'cost'
pred_out = probs > 0.5
mis_cost = T.sum(T.neq(y.flatten(), pred_out.flatten()))
mis_cost.name = 'MisclassificationRate'
return mlp, fine_tuner, cost, mis_cost
def create_vae(x=None, batch=batch_size):
x = T.matrix('features') if x is None else x
x = x / 255.
encoder = MLP(
activations=[Rectifier(), Logistic()],
dims=[img_dim**2, hidden_dim, 2*latent_dim],
weights_init=IsotropicGaussian(std=0.01, mean=0),
biases_init=Constant(0.01),
name='encoder'
)
encoder.initialize()
z_param = encoder.apply(x)
z_mean, z_log_std = z_param[:,latent_dim:], z_param[:,:latent_dim]
z = Sampling(theano_seed=seed).apply([z_mean, z_log_std], batch=batch_size)
decoder = MLP(
activations=[Rectifier(), Logistic()],
dims=[latent_dim, hidden_dim, img_dim**2],
weights_init=IsotropicGaussian(std=0.01, mean=0),
biases_init=Constant(0.01),
name='decoder'
)
decoder.initialize()
x_reconstruct = decoder.apply(z)
cost = VAEloss().apply(x, x_reconstruct, z_mean, z_log_std)
cost.name = 'vae_cost'
return cost
def create_model_bricks():
convnet = ConvolutionalSequence(
layers=[
Convolutional(
filter_size=(4, 4),
num_filters=32,
name='conv1'),
SpatialBatchNormalization(name='batch_norm1'),
Rectifier(),
Convolutional(
filter_size=(3, 3),
step=(2, 2),
num_filters=32,
name='conv2'),
SpatialBatchNormalization(name='batch_norm2'),
Rectifier(),
Convolutional(
filter_size=(4, 4),
num_filters=64,
name='conv3'),
SpatialBatchNormalization(name='batch_norm3'),
Rectifier(),
Convolutional(
filter_size=(3, 3),
step=(2, 2),
num_filters=64,
name='conv4'),
SpatialBatchNormalization(name='batch_norm4'),
Rectifier(),
Convolutional(
filter_size=(3, 3),
num_filters=128,
name='conv5'),
SpatialBatchNormalization(name='batch_norm5'),
Rectifier(),
Convolutional(
filter_size=(3, 3),
step=(2, 2),
num_filters=128,
name='conv6'),
SpatialBatchNormalization(name='batch_norm6'),
Rectifier(),
],
num_channels=3,
image_size=(64, 64),
use_bias=False,
weights_init=IsotropicGaussian(0.033),
biases_init=Constant(0),
name='convnet')
convnet.initialize()
mlp = BatchNormalizedMLP(
activations=[Rectifier(), Logistic()],
dims=[numpy.prod(convnet.get_dim('output')), 1000, 40],
weights_init=IsotropicGaussian(0.033),
biases_init=Constant(0),
name='mlp')
mlp.initialize()
return convnet, mlp
def __init__(self, dim, mini_dim, summary_dim, **kwargs):
super(RNNwMini, self).__init__(**kwargs)
self.dim = dim
self.mini_dim = mini_dim
self.summary_dim = summary_dim
self.recurrent_layer = SimpleRecurrent(
dim=self.summary_dim,
activation=Rectifier(),
name='recurrent_layer',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
self.mini_recurrent_layer = SimpleRecurrent(
dim=self.mini_dim,
activation=Rectifier(),
name='mini_recurrent_layer',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
self.mini_to_main = Linear(self.dim + self.mini_dim,
self.summary_dim,
name='mini_to_main',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
self.children = [
self.recurrent_layer, self.mini_recurrent_layer, self.mini_to_main
]
def __init__(self, feature_dim, memory_dim, fc1_dim, fc2_dim):
self.W = Linear(input_dim=feature_dim,
output_dim=memory_dim * 4,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
use_bias=False,
name='seqDecoder_W')
self.GRU_A = LSTM(feature_dim,
name='seqDecoder_A',
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
self.GRU_B = LSTM(memory_dim,
name='seqDecoder_B',
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
self.W.initialize()
self.GRU_A.initialize()
self.GRU_B.initialize()
self.fc1 = Linear(input_dim=memory_dim,
output_dim=fc1_dim,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
name='fc1')
self.fc2 = Linear(input_dim=fc1_dim,
output_dim=fc2_dim,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
name='fc2')
self.fc1.initialize()
self.fc2.initialize()
def __init__(self,
input_dim,
output_dim,
lstm_dim,
print_intermediate=False,
print_attrs=['__str__'],
**kwargs):
super(LinearLSTM, self).__init__(**kwargs)
self.x_to_h = Linear(input_dim,
lstm_dim * 4,
name='x_to_h',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
self.lstm = LSTM(lstm_dim,
name='lstm',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
self.h_to_o = Linear(lstm_dim,
output_dim,
name='h_to_o',
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0))
self.children = [self.x_to_h, self.lstm, self.h_to_o]
self.print_intermediate = print_intermediate
self.print_attrs = print_attrs
def getRnnGenerator(vocab_size,hidden_dim,input_dim=512):
"""
"Apply" the RNN to the input x
For initializing the network, the vocab size needs to be known
Default of the hidden layer is set tot 512 like Karpathy
"""
generator = SequenceGenerator(
Readout(readout_dim = vocab_size,
source_names = ["states"], # transition.apply.states ???
emitter = SoftmaxEmitter(name="emitter"),
feedback_brick = LookupFeedback(
vocab_size,
input_dim,
name = 'feedback'
),
name = "readout"
),
MySimpleRecurrent(
name = "transition",
activation = Tanh(),
dim = hidden_dim
),
weights_init = IsotropicGaussian(0.01),
biases_init = Constant(0),
name = "generator"
)
generator.push_initialization_config()
generator.transition.weights_init = IsotropicGaussian(0.01)
generator.initialize()
return generator
def setup_model():
# shape: T x B x F
input_ = T.tensor3('features')
# shape: B
target = T.lvector('targets')
model = LSTMAttention(input_dim=10000,
dim=500,
mlp_hidden_dims=[2000, 500, 4],
batch_size=100,
image_shape=(100, 100),
patch_shape=(28, 28),
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
model.initialize()
h, c = model.apply(input_)
classifier = MLP([Rectifier(), Softmax()], [500, 100, 10],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
classifier.initialize()
probabilities = classifier.apply(h[-1])
cost = CategoricalCrossEntropy().apply(target, probabilities)
error_rate = MisclassificationRate().apply(target, probabilities)
return cost, error_rate
def __init__(self, **kwargs):
children = []
self.layers_numerical = []
self.layers_numerical.append(
Linear(name='input_to_numerical_linear',
input_dim=5000,
output_dim=17,
weights_init=IsotropicGaussian(),
biases_init=Constant(1)))
self.layers_categorical = []
self.layers_categorical.append(
Linear(name='input_to_categorical_linear',
input_dim=5000,
output_dim=24016,
weights_init=IsotropicGaussian(),
biases_init=Constant(1)))
self.layers_categorical.append(
Logistic(name='input_to_categorical_sigmoid'))
children += self.layers_numerical
children += self.layers_categorical
kwargs.setdefault('children', []).extend(children)
super(build_top_mlp, self).__init__(**kwargs)
def build_mlp(features_cat, features_int, labels):
mlp_int = MLP(activations=[Rectifier(), Rectifier()],
dims=[19, 50, 50],
weights_init=IsotropicGaussian(),
biases_init=Constant(0),
name='mlp_interval')
mlp_int.initialize()
mlp_cat = MLP(activations=[Logistic()],
dims=[320, 50],
weights_init=IsotropicGaussian(),
biases_init=Constant(0),
name='mlp_categorical')
mlp_cat.initialize()
mlp = MLP(activations=[Rectifier(), None],
dims=[50, 50, 1],
weights_init=IsotropicGaussian(),
biases_init=Constant(0))
mlp.initialize()
gated = mlp_cat.apply(features_cat) * mlp_int.apply(features_int)
prediction = mlp.apply(gated)
cost = MAPECost().apply(prediction, labels)
cg = ComputationGraph(cost)
print cg.variables
cg_dropout1 = apply_dropout(cg, [VariableFilter(roles=[OUTPUT])(cg.variables)[1], VariableFilter(roles=[OUTPUT])(cg.variables)[3]], .2)
cost_dropout1 = cg_dropout1.outputs[0]
return cost_dropout1, cg_dropout1.parameters, cost
def _build_bricks(self, *args, **kwargs):
# Build lookup tables
self.word_embed = self._embed(len(self.dataset.word2index),
self.config.word_embed_dim,
name='word_embed')
self.hashtag_embed = self._embed(len(self.dataset.hashtag2index),
self.config.lstm_dim,
name='hashtag_embed')
# Build text encoder
self.mlstm_ins = Linear(input_dim=self.config.word_embed_dim,
output_dim=4 * self.config.lstm_dim,
name='mlstm_in')
self.mlstm_ins.weights_init = IsotropicGaussian(
std=numpy.sqrt(2) /
numpy.sqrt(self.config.word_embed_dim + self.config.lstm_dim))
self.mlstm_ins.biases_init = Constant(0)
self.mlstm_ins.initialize()
self.mlstm = MLSTM(self.config.lstm_time,
self.config.lstm_dim,
shared=False)
self.mlstm.weights_init = IsotropicGaussian(
std=numpy.sqrt(2) /
numpy.sqrt(self.config.word_embed_dim + self.config.lstm_dim))
self.mlstm.biases_init = Constant(0)
self.mlstm.initialize()
self.hashtag2word = MLP(
activations=[Tanh('hashtag2word_tanh')],
dims=[self.config.lstm_dim, self.config.word_embed_dim],
name='hashtag2word_mlp')
self.hashtag2word.weights_init = IsotropicGaussian(
std=1 / numpy.sqrt(self.config.word_embed_dim))
self.hashtag2word.biases_init = Constant(0)
self.hashtag2word.initialize()
self.hashtag2word_bias = Bias(dim=1, name='hashtag2word_bias')
self.hashtag2word_bias.biases_init = Constant(0)
self.hashtag2word_bias.initialize()
#Build character embedding
self.char_embed = self._embed(len(self.dataset.char2index),
self.config.char_embed_dim,
name='char_embed')
# Build sparse word encoder
self.rnn_ins = Linear(input_dim=self.config.char_embed_dim,
output_dim=self.config.word_embed_dim,
name='rnn_in')
self.rnn_ins.weights_init = IsotropicGaussian(
std=numpy.sqrt(2) / numpy.sqrt(self.config.char_embed_dim +
self.config.word_embed_dim))
self.rnn_ins.biases_init = Constant(0)
self.rnn_ins.initialize()
self.rnn = SimpleRecurrent(dim=self.config.word_embed_dim,
activation=Tanh())
self.rnn.weights_init = IsotropicGaussian(
std=1 / numpy.sqrt(self.config.word_embed_dim))
self.rnn.initialize()
def create_model(self):
input_dim = self.input_dim
x = self.x
y = self.y
p = self.p
mask = self.mask
hidden_dim = self.hidden_dim
embedding_dim = self.embedding_dim
lookup = LookupTable(self.dict_size,
embedding_dim,
weights_init=IsotropicGaussian(0.001),
name='LookupTable')
x_to_h = Linear(embedding_dim,
hidden_dim * 4,
name='x_to_h',
weights_init=IsotropicGaussian(0.001),
biases_init=Constant(0.0))
lstm = LSTM(hidden_dim,
name='lstm',
weights_init=IsotropicGaussian(0.001),
biases_init=Constant(0.0))
h_to_o = MLP([Logistic()], [hidden_dim, 1],
weights_init=IsotropicGaussian(0.001),
biases_init=Constant(0),
name='h_to_o')
lookup.initialize()
x_to_h.initialize()
lstm.initialize()
h_to_o.initialize()
embed = lookup.apply(x).reshape(
(x.shape[0], x.shape[1], self.embedding_dim))
embed.name = "embed_vec"
x_transform = x_to_h.apply(embed.transpose(1, 0, 2))
x_transform.name = "Transformed X"
self.lookup = lookup
self.x_to_h = x_to_h
self.lstm = lstm
self.h_to_o = h_to_o
#if mask is None:
h, c = lstm.apply(x_transform)
#else:
#h, c = lstm.apply(x_transform, mask=mask)
h.name = "hidden_state"
c.name = "cell state"
# only values of hidden units of the last timeframe are used for
# the classification
indices = T.sum(mask, axis=0) - 1
rel_hid = h[indices, T.arange(h.shape[1])]
out = self.h_to_o.apply(rel_hid)
probs = out
return probs
def test_fully_layer():
batch_size=2
x = T.tensor4();
y = T.ivector()
V = 200
layer_conv = Convolutional(filter_size=(5,5),num_filters=V,
name="toto",
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0.0))
# try with no bias
activation = Rectifier()
pool = MaxPooling(pooling_size=(2,2))
convnet = ConvolutionalSequence([layer_conv, activation, pool], num_channels=15,
image_size=(10,10),
name="conv_section")
convnet.push_allocation_config()
convnet.initialize()
output=convnet.apply(x)
batch_size=output.shape[0]
output_dim=np.prod(convnet.get_dim('output'))
result_conv = output.reshape((batch_size, output_dim))
mlp=MLP(activations=[Rectifier().apply], dims=[output_dim, 10],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0.0))
mlp.initialize()
output=mlp.apply(result_conv)
cost = T.mean(Softmax().categorical_cross_entropy(y.flatten(), output))
cg = ComputationGraph(cost)
W = VariableFilter(roles=[WEIGHT])(cg.variables)
B = VariableFilter(roles=[BIAS])(cg.variables)
W = W[0]; b = B[0]
inputs_fully = VariableFilter(roles=[INPUT], bricks=[Linear])(cg)
outputs_fully = VariableFilter(roles=[OUTPUT], bricks=[Linear])(cg)
var_input=inputs_fully[0]
var_output=outputs_fully[0]
[d_W,d_S,d_b] = T.grad(cost, [W, var_output, b])
d_b = d_b.dimshuffle(('x',0))
d_p = T.concatenate([d_W, d_b], axis=0)
x_value = 1e3*np.random.ranf((2,15, 10, 10))
f = theano.function([x,y], [var_input, d_S, d_p], allow_input_downcast=True, on_unused_input='ignore')
A, B, C= f(x_value, [5, 0])
A = np.concatenate([A, np.ones((2,1))], axis=1)
print 'A', A.shape
print 'B', B.shape
print 'C', C.shape
print lin.norm(C - np.dot(np.transpose(A), B), 'fro')
return
"""
def build_mlp(features_car_cat, features_car_int, features_nocar_cat,
features_nocar_int, features_cp, features_hascar, means, labels):
mlp_car = MLP(activations=[Rectifier(), Rectifier(), None],
dims=[8 + 185, 200, 200, 1],
weights_init=IsotropicGaussian(.1),
biases_init=Constant(0),
name='mlp_interval_car')
mlp_car.initialize()
mlp_nocar = MLP(activations=[Rectifier(), Rectifier(), None],
dims=[5 + 135, 200, 200, 1],
weights_init=IsotropicGaussian(.1),
biases_init=Constant(0),
name='mlp_interval_nocar')
mlp_nocar.initialize()
feature_car = tensor.concatenate((features_car_cat, features_car_int),
axis=1)
feature_nocar = tensor.concatenate(
(features_nocar_cat, features_nocar_int), axis=1)
prediction = mlp_nocar.apply(feature_nocar)
# gating with the last feature : does the dude own a car
prediction += tensor.addbroadcast(features_hascar,
1) * mlp_car.apply(feature_car)
prediction_loc, _, _, _, = \
build_mlp_onlyloc(features_car_cat, features_car_int,
features_nocar_cat, features_nocar_int,
features_cp, features_hascar,
means, labels)
prediction += prediction_loc
# add crm
mlp_crm = MLP(activations=[None],
dims=[1, 1],
weights_init=IsotropicGaussian(.1),
biases_init=Constant(0),
name='mlp_crm')
mlp_crm.initialize()
crm = features_nocar_int[:, 0][:, None]
prediction = prediction * mlp_crm.apply(crm)
cost = MAPECost().apply(labels, prediction)
cg = ComputationGraph(cost)
input_var = VariableFilter(roles=[INPUT])(cg.variables)
print input_var
cg_dropout1 = apply_dropout(cg, [input_var[6], input_var[7]], .4)
cost_dropout1 = cg_dropout1.outputs[0]
return prediction, cost_dropout1, cg_dropout1.parameters, cost
def __init__(self, word_dim, visual_dim, joint_dim):
self.word_embed = Linear(word_dim,
joint_dim,
name='word_to_joint',
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
self.visual_embed = Linear(visual_dim,
joint_dim,
name='visual_to_joint',
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
self.word_embed.initialize()
self.visual_embed.initialize()
def main(save_to, num_batches, continue_=False):
mlp = MLP([Tanh(), Identity()], [1, 10, 1],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
seed=1)
mlp.initialize()
x = tensor.vector('numbers')
y = tensor.vector('roots')
cost = SquaredError().apply(y[:, None], mlp.apply(x[:, None]))
cost.name = "cost"
main_loop = MainLoop(
GradientDescent(cost=cost,
params=ComputationGraph(cost).parameters,
step_rule=Scale(learning_rate=0.001)),
get_data_stream(range(100)),
model=Model(cost),
extensions=([LoadFromDump(save_to)] if continue_ else []) + [
Timing(),
FinishAfter(after_n_batches=num_batches),
DataStreamMonitoring(
[cost], get_data_stream(range(100, 200)), prefix="test"),
TrainingDataMonitoring([cost], after_epoch=True),
Dump(save_to),
Printing()
])
main_loop.run()
return main_loop
def build_mlp(features_int, features_cat, labels, labels_mean):
inputs = tensor.concatenate([features_int, features_cat], axis=1)
mlp = MLP(activations=[Rectifier(),
Rectifier(),
Rectifier(), None],
dims=[337, 800, 1200, 1],
weights_init=IsotropicGaussian(),
biases_init=Constant(1))
mlp.initialize()
prediction = mlp.apply(inputs)
cost = MAPECost().apply(prediction, labels, labels_mean)
cg = ComputationGraph(cost)
#cg_dropout0 = apply_dropout(cg, [VariableFilter(roles=[INPUT])(cg.variables)[1]], .2)
cg_dropout1 = apply_dropout(cg, [
VariableFilter(roles=[OUTPUT])(cg.variables)[1],
VariableFilter(roles=[OUTPUT])(cg.variables)[3],
VariableFilter(roles=[OUTPUT])(cg.variables)[5]
], .2)
cost_dropout1 = cg_dropout1.outputs[0]
return cost_dropout1, cg_dropout1.parameters, cost #cost, cg.parameters, cost #
def construct_model(input_dim, output_dim):
# Construct the model
r = tensor.fmatrix('r')
x = tensor.fmatrix('x')
y = tensor.ivector('y')
# input_dim must be nr
mlp = MLP(activations=activation_functions,
dims=[input_dim] + hidden_dims + [2])
weights = mlp.apply(r)
final = tensor.dot(x, weights)
cost = Softmax().categorical_cross_entropy(y, final).mean()
pred = final.argmax(axis=1)
error_rate = tensor.neq(y, pred).mean()
# Initialize parameters
for brick in [mlp]:
brick.weights_init = IsotropicGaussian(0.01)
brick.biases_init = Constant(0.001)
brick.initialize()
# apply noise
cg = ComputationGraph([cost, error_rate])
noise_vars = VariableFilter(roles=[WEIGHT])(cg)
apply_noise(cg, noise_vars, noise_std)
[cost, error_rate] = cg.outputs
return cost, error_rate
def create_model(self, x, y, input_dim, tol=10e-5):
# Create the output of the MLP
mlp = MLP(
[Rectifier(), Rectifier(), Logistic()], [input_dim, 100, 100, 1],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
mlp.initialize()
probs = mlp.apply(x)
y = y.dimshuffle(0, 'x')
# Create the if-else cost function
true_p = (T.sum(y * probs) + tol) * 1.0 / (T.sum(y) + tol)
true_n = (T.sum((1 - y) * (1 - probs)) + tol) * \
1.0 / (T.sum(1 - y) + tol)
#p = (T.sum(y) + tol) / (y.shape[0] + tol)
theta = (1 - self.p) / self.p
numerator = (1 + self.beta**2) * true_p
denominator = self.beta**2 + theta + true_p - theta * true_n
Fscore = numerator / denominator
cost = -1 * Fscore
cost.name = "cost"
return mlp, cost, probs
def setup_ff_network(in_dim, out_dim, num_layers, num_neurons):
"""Setup a feedforward neural network.
Parameters
----------
in_dim : int
input dimension of network
out_dim : int
output dimension of network
num_layers : int
number of hidden layers
num_neurons : int
number of neurons of each layer
Returns
-------
net : object
network structure
"""
activations = [Rectifier()]
dims = [in_dim]
for i in xrange(num_layers):
activations.append(Rectifier())
dims.append(num_neurons)
dims.append(out_dim)
net = MLP(activations=activations,
dims=dims,
weights_init=IsotropicGaussian(),
biases_init=Constant(0.01))
return net
def test_sequence_generator():
# Disclaimer: here we only check shapes, not values.
output_dim = 1
dim = 20
batch_size = 30
n_steps = 10
transition = GatedRecurrent(
name="transition", activation=Tanh(), dim=dim,
weights_init=Orthogonal())
generator = SequenceGenerator(
LinearReadout(readout_dim=output_dim, source_names=["states"],
emitter=TestEmitter(name="emitter"), name="readout"),
transition,
weights_init=IsotropicGaussian(0.01), biases_init=Constant(0),
name="generator")
generator.initialize()
y = tensor.tensor3('y')
mask = tensor.matrix('mask')
costs = generator.cost(y, mask)
assert costs.ndim == 2
costs_val = theano.function([y, mask], [costs])(
numpy.zeros((n_steps, batch_size, output_dim), dtype=floatX),
numpy.ones((n_steps, batch_size), dtype=floatX))[0]
assert costs_val.shape == (n_steps, batch_size)
states, outputs, costs = [variable.eval() for variable in
generator.generate(
iterate=True, batch_size=batch_size,
n_steps=n_steps)]
assert states.shape == (n_steps, batch_size, dim)
assert outputs.shape == (n_steps, batch_size, output_dim)
assert costs.shape == (n_steps, batch_size)
def build_mlp(features_car_cat, features_car_int, features_nocar_cat,
features_nocar_int, features_cp, features_hascar, means, labels):
prediction, _, _, _, = \
build_mlp_onlyloc(features_car_cat, features_car_int,
features_nocar_cat, features_nocar_int, features_cp, features_hascar,
means, labels)
mlp_crm = MLP(activations=[None],
dims=[1, 1],
weights_init=IsotropicGaussian(.1),
biases_init=Constant(0),
name='mlp_crm')
mlp_crm.initialize()
crm = features_nocar_int[:, 0][:, None]
prediction = prediction * mlp_crm.apply(crm)
cost = MAPECost().apply(labels, prediction)
cg = ComputationGraph(cost)
input_var = VariableFilter(roles=[INPUT])(cg.variables)
print input_var
cg_dropout = apply_dropout(cg, [input_var[7], input_var[5]], .4)
cost_dropout = cg_dropout.outputs[0]
return prediction, cost_dropout, cg_dropout.parameters, cost
def build_conv_layers(self, image=None):
if image is None:
image = T.ftensor4('spectrogram')
else:
image = image
conv_list = []
for layer in range(self.layers):
layer_param = self.params[layer]
conv_layer = Convolutional(layer_param[0], layer_param[1],
layer_param[2])
pool_layer = MaxPooling(layer_param[3])
conv_layer.name = "convolution" + str(layer)
pool_layer.name = "maxpooling" + str(layer)
conv_list.append(conv_layer)
conv_list.append(pool_layer)
conv_list.append(Rectifier())
conv_seq = ConvolutionalSequence(conv_list,
self.params[0][2],
image_size=self.image_size,
weights_init=IsotropicGaussian(
std=0.5, mean=0),
biases_init=Constant(0))
conv_seq._push_allocation_config()
conv_seq.initialize()
out = conv_seq.apply(image)
return out, conv_seq.get_dim('output')
def test_pylearn2_trainin():
# Construct the model
mlp = MLP(activations=[Sigmoid(), Sigmoid()],
dims=[784, 100, 784],
weights_init=IsotropicGaussian(),
biases_init=Constant(0.01))
mlp.initialize()
cost = SquaredError()
block_cost = BlocksCost(cost)
block_model = BlocksModel(mlp, (VectorSpace(dim=784), 'features'))
# Load the data
rng = numpy.random.RandomState(14)
train_dataset = random_dense_design_matrix(rng, 1024, 784, 10)
valid_dataset = random_dense_design_matrix(rng, 1024, 784, 10)
# Silence Pylearn2's logger
logger = logging.getLogger(pylearn2.__name__)
logger.setLevel(logging.ERROR)
# Training algorithm
sgd = SGD(learning_rate=0.01,
cost=block_cost,
batch_size=128,
monitoring_dataset=valid_dataset)
train = Train(train_dataset, block_model, algorithm=sgd)
train.main_loop(time_budget=3)
def __init__(self, image_shape, patch_shape, hidden_dim,
n_spatial_dims, whatwhere_interaction, prefork_area_transform,
postmerge_area_transform, patch_transform, batch_normalize,
response_transform, location_std, scale_std, cutoff,
batched_window, initargs, emitter, **kwargs):
self.rnn = LSTM(activation=Tanh(),
dim=hidden_dim,
name="recurrent",
weights_init=IsotropicGaussian(1e-4),
biases_init=Constant(0))
self.locator = masonry.Locator(hidden_dim, n_spatial_dims,
area_transform=prefork_area_transform,
location_std=location_std,
scale_std=scale_std,
**initargs)
self.cropper = crop.LocallySoftRectangularCropper(
n_spatial_dims=n_spatial_dims,
image_shape=image_shape, patch_shape=patch_shape,
kernel=crop.Gaussian(), cutoff=cutoff,
batched_window=batched_window)
self.merger = masonry.Merger(
patch_transform=patch_transform,
area_transform=postmerge_area_transform,
response_transform=response_transform,
n_spatial_dims=n_spatial_dims,
whatwhere_interaction=whatwhere_interaction,
batch_normalize=batch_normalize,
**initargs)
self.attention = masonry.SpatialAttention(
self.locator, self.cropper, self.merger,
name="sa")
self.emitter = emitter
self.model = masonry.RecurrentAttentionModel(
self.rnn, self.attention, self.emitter,
name="ram")
def main(save_to, num_epochs):
mlp = MLP([Tanh(), Softmax()], [784, 100, 10],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0))
mlp.initialize()
x = tensor.matrix('features')
y = tensor.lmatrix('targets')
probs = mlp.apply(tensor.flatten(x, outdim=2))
cost = CategoricalCrossEntropy().apply(y.flatten(), probs)
error_rate = MisclassificationRate().apply(y.flatten(), probs)
cg = ComputationGraph([cost])
W1, W2 = VariableFilter(roles=[WEIGHT])(cg.variables)
cost = cost + .00005 * (W1**2).sum() + .00005 * (W2**2).sum()
cost.name = 'final_cost'
mnist_train = MNIST(("train", ))
mnist_test = MNIST(("test", ))
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Scale(learning_rate=0.1))
extensions = [
Timing(),
FinishAfter(after_n_epochs=num_epochs),
DataStreamMonitoring([cost, error_rate],
Flatten(DataStream.default_stream(
mnist_test,
iteration_scheme=SequentialScheme(
mnist_test.num_examples, 500)),
which_sources=('features', )),
prefix="test"),
TrainingDataMonitoring([
cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)
],
prefix="train",
after_epoch=True),
Checkpoint(save_to),
Printing()
]
if BLOCKS_EXTRAS_AVAILABLE:
extensions.append(
Plot('MNIST example',
channels=[[
'test_final_cost',
'test_misclassificationrate_apply_error_rate'
], ['train_total_gradient_norm']]))
main_loop = MainLoop(algorithm,
Flatten(DataStream.default_stream(
mnist_train,
iteration_scheme=SequentialScheme(
mnist_train.num_examples, 50)),
which_sources=('features', )),
model=Model(cost),
extensions=extensions)
main_loop.run()
def __init__(self, x_dim, hidden_layers, hidden_act, z_dim, batch_norm=False, l2reg=1e-3, **kwargs):
super(VAE, self).__init__([], [], **kwargs)
self.l2reg = l2reg
inits = {
'weights_init': IsotropicGaussian(std=0.1),
#'weights_init': RWSInitialization(factor=1.),
'biases_init': Constant(0.0),
}
if batch_norm:
mlp_class = BatchNormalizedMLP
else:
mlp_class = MLP
hidden_act = [hidden_act] * len(hidden_layers)
q_mlp = mlp_class(hidden_act, [x_dim] + hidden_layers, **inits)
p_mlp = mlp_class(hidden_act + [Logistic()], [z_dim] + hidden_layers + [x_dim], **inits)
self.q = GaussianLayer(z_dim, q_mlp, **inits)
self.p = BernoulliLayer(p_mlp, **inits)
self.prior_log_sigma = numpy.zeros(z_dim) #
self.prior_mu = numpy.zeros(z_dim) #
self.children = [self.p, self.q]
