def sampling_step(g_noise, states, samples_step):
embedding_step = linear_embedding.apply(samples_step)
next_states = rnn.apply(inputs=embedding_step,
states=states,
iterate=False)
probs_step = softmax(score_layer.apply(next_states))
next_samples = (tensor.log(probs_step)
+ g_noise).argmax(axis=-1)
return next_states, next_samples
def get_probs(self, features):
"""Output the probability of being a positive class
Parameters
----------
features : :class:`~tensor.TensorVariable`
The features that you consider as input.
Must have shape (batch_size, input_dim).
Returns
-------
probs : :class:`~tensor.TensorVariable`
The probabilities of each example to belong to
the positive class. Must have shape (batch_size, 1)
"""
return softmax(features.dot(self.W) + self.b)
def get_probs(self, features):
"""Output the probability of being a positive class
Parameters
----------
features : :class:`~tensor.TensorVariable`
The features that you consider as input.
Must have shape (batch_size, input_dim).
Returns
-------
probs : :class:`~tensor.TensorVariable`
The probabilities of each example to belong to
the positive class. Must have shape (batch_size, 1)
"""
return softmax(features.dot(self.W) + self.b)
def get_probs(self, features):
"""Output the probability of belonging to a class
Parameters
----------
features : :class:`~tensor.TensorVariable`
The features that you consider as input.
Must have shape (batch_size, input_dim).
Returns
-------
probs : :class:`~tensor.TensorVariable`
The probabilities of each example of belonging to
each class. Must have shape (batch_size, n_classes)
"""
out = features;
for n in xrange(len(self.neural_arch)-1):
out = softmax(out.dot(self.W[n]) + self.b[n])
return out
def get_probs(self, features):
"""Output the probability of belonging to a class
Parameters
----------
features : :class:`~tensor.TensorVariable`
The features that you consider as input.
Must have shape (batch_size, input_dim).
Returns
-------
probs : :class:`~tensor.TensorVariable`
The probabilities of each example of belonging to
each class. Must have shape (batch_size, n_classes)
"""
out = features
for W, b in zip(self.W, self.b):
out = out.dot(W) + b
if W != self.W[-1]:
out = (out > 0.) * (out)
return softmax(out)
def main(num_epochs=100):
x = tensor.matrix('features')
m = tensor.matrix('features_mask')
x_int = x.astype(dtype='int32').T
train_dataset = TextFile('inspirational.txt')
train_dataset.indexables[0] = numpy.array(sorted(
train_dataset.indexables[0], key=len
))
n_voc = len(train_dataset.dict.keys())
init_probs = numpy.array(
[sum(filter(lambda idx:idx == w,
[s[0] for s in train_dataset.indexables[
train_dataset.sources.index('features')]]
)) for w in xrange(n_voc)],
dtype=theano.config.floatX
)
init_probs = init_probs / init_probs.sum()
n_h = 100
linear_embedding = LookupTable(
length=n_voc,
dim=n_h,
weights_init=Uniform(std=0.01),
biases_init=Constant(0.)
)
linear_embedding.initialize()
lstm_biases = numpy.zeros(4 * n_h).astype(dtype=theano.config.floatX)
lstm_biases[n_h:(2 * n_h)] = 4.
rnn = SimpleRecurrent(
dim=n_h,
activation=Tanh(),
weights_init=Uniform(std=0.01),
biases_init=Constant(0.)
)
rnn.initialize()
score_layer = Linear(
input_dim=n_h,
output_dim=n_voc,
weights_init=Uniform(std=0.01),
biases_init=Constant(0.)
)
score_layer.initialize()
embedding = (linear_embedding.apply(x_int[:-1])
* tensor.shape_padright(m.T[1:]))
rnn_out = rnn.apply(inputs=embedding, mask=m.T[1:])
probs = softmax(
sequence_map(score_layer.apply, rnn_out, mask=m.T[1:])[0]
)
idx_mask = m.T[1:].nonzero()
cost = CategoricalCrossEntropy().apply(
x_int[1:][idx_mask[0], idx_mask[1]],
probs[idx_mask[0], idx_mask[1]]
)
cost.name = 'cost'
misclassification = MisclassificationRate().apply(
x_int[1:][idx_mask[0], idx_mask[1]],
probs[idx_mask[0], idx_mask[1]]
)
misclassification.name = 'misclassification'
cg = ComputationGraph([cost])
params = cg.parameters
algorithm = GradientDescent(
cost=cost,
params=params,
step_rule=Adam()
)
train_data_stream = Padding(
data_stream=DataStream(
dataset=train_dataset,
iteration_scheme=BatchwiseShuffledScheme(
examples=train_dataset.num_examples,
batch_size=10,
)
),
mask_sources=('features',)
)
model = Model(cost)
extensions = []
extensions.append(Timing())
extensions.append(FinishAfter(after_n_epochs=num_epochs))
extensions.append(TrainingDataMonitoring(
[cost, misclassification],
prefix='train',
after_epoch=True))
batch_size = 10
length = 30
trng = MRG_RandomStreams(18032015)
u = trng.uniform(size=(length, batch_size, n_voc))
gumbel_noise = -tensor.log(-tensor.log(u))
init_samples = (tensor.log(init_probs).dimshuffle(('x', 0))
+ gumbel_noise[0]).argmax(axis=-1)
init_states = rnn.initial_state('states', batch_size)
def sampling_step(g_noise, states, samples_step):
embedding_step = linear_embedding.apply(samples_step)
next_states = rnn.apply(inputs=embedding_step,
states=states,
iterate=False)
probs_step = softmax(score_layer.apply(next_states))
next_samples = (tensor.log(probs_step)
+ g_noise).argmax(axis=-1)
return next_states, next_samples
[_, samples], _ = theano.scan(
fn=sampling_step,
sequences=[gumbel_noise[1:]],
outputs_info=[init_states, init_samples]
)
sampling = theano.function([], samples.owner.inputs[0].T)
plotters = []
plotters.append(Plotter(
channels=[['train_cost', 'train_misclassification']],
titles=['Costs']))
extensions.append(PlotManager('Language modelling example',
plotters=plotters,
after_epoch=True,
after_training=True))
extensions.append(Printing())
extensions.append(PrintSamples(sampler=sampling,
voc=train_dataset.inv_dict))
main_loop = MainLoop(model=model,
data_stream=train_data_stream,
algorithm=algorithm,
extensions=extensions)
main_loop.run()
