def test_sequence_generator():
# Disclaimer: here we only check shapes, not values.
output_dim = 1
dim = 20
batch_size = 30
n_steps = 10
transition = GatedRecurrent(
name="transition", activation=Tanh(), dim=dim,
weights_init=Orthogonal())
generator = SequenceGenerator(
LinearReadout(readout_dim=output_dim, source_names=["states"],
emitter=TestEmitter(name="emitter"), name="readout"),
transition,
weights_init=IsotropicGaussian(0.01), biases_init=Constant(0),
name="generator")
generator.initialize()
y = tensor.tensor3('y')
mask = tensor.matrix('mask')
costs = generator.cost(y, mask)
assert costs.ndim == 2
costs_val = theano.function([y, mask], [costs])(
numpy.zeros((n_steps, batch_size, output_dim), dtype=floatX),
numpy.ones((n_steps, batch_size), dtype=floatX))[0]
assert costs_val.shape == (n_steps, batch_size)
states, outputs, costs = [variable.eval() for variable in
generator.generate(
iterate=True, batch_size=batch_size,
n_steps=n_steps)]
assert states.shape == (n_steps, batch_size, dim)
assert outputs.shape == (n_steps, batch_size, output_dim)
assert costs.shape == (n_steps, batch_size)
def getRnnGenerator(vocab_size,hidden_dim,input_dim=512):
"""
"Apply" the RNN to the input x
For initializing the network, the vocab size needs to be known
Default of the hidden layer is set tot 512 like Karpathy
"""
generator = SequenceGenerator(
Readout(readout_dim = vocab_size,
source_names = ["states"], # transition.apply.states ???
emitter = SoftmaxEmitter(name="emitter"),
feedback_brick = LookupFeedback(
vocab_size,
input_dim,
name = 'feedback'
),
name = "readout"
),
MySimpleRecurrent(
name = "transition",
activation = Tanh(),
dim = hidden_dim
),
weights_init = IsotropicGaussian(0.01),
biases_init = Constant(0),
name = "generator"
)
generator.push_initialization_config()
generator.transition.weights_init = IsotropicGaussian(0.01)
generator.initialize()
return generator
def test_sequence_generator():
# Disclaimer: here we only check shapes, not values.
output_dim = 1
dim = 20
batch_size = 30
n_steps = 10
transition = GatedRecurrent(
name="transition", activation=Tanh(), dim=dim,
weights_init=Orthogonal())
generator = SequenceGenerator(
LinearReadout(readout_dim=output_dim, source_names=["states"],
emitter=TestEmitter(name="emitter"), name="readout"),
transition,
weights_init=IsotropicGaussian(0.01), biases_init=Constant(0),
name="generator")
generator.initialize()
y = tensor.tensor3('y')
mask = tensor.matrix('mask')
costs = generator.cost(y, mask)
assert costs.ndim == 2
costs_val = theano.function([y, mask], [costs])(
numpy.zeros((n_steps, batch_size, output_dim), dtype=floatX),
numpy.ones((n_steps, batch_size), dtype=floatX))[0]
assert costs_val.shape == (n_steps, batch_size)
states, outputs, costs = [variable.eval() for variable in
generator.generate(
iterate=True, batch_size=batch_size,
n_steps=n_steps)]
assert states.shape == (n_steps, batch_size, dim)
assert outputs.shape == (n_steps, batch_size, output_dim)
assert costs.shape == (n_steps, batch_size)
def test_integer_sequence_generator():
# Disclaimer: here we only check shapes, not values.
readout_dim = 5
feedback_dim = 3
dim = 20
batch_size = 30
n_steps = 10
transition = GatedRecurrent(name="transition",
activation=Tanh(),
dim=dim,
weights_init=Orthogonal())
generator = SequenceGenerator(LinearReadout(
readout_dim=readout_dim,
source_names=["states"],
emitter=SoftmaxEmitter(name="emitter"),
feedbacker=LookupFeedback(readout_dim, feedback_dim),
name="readout"),
transition,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
name="generator")
generator.initialize()
y = tensor.lmatrix('y')
mask = tensor.matrix('mask')
costs = generator.cost(y, mask)
assert costs.ndim == 2
costs_val = theano.function([y, mask],
[costs])(numpy.zeros((n_steps, batch_size),
dtype='int64'),
numpy.ones((n_steps, batch_size),
dtype=floatX))[0]
assert costs_val.shape == (n_steps, batch_size)
states, outputs, costs = generator.generate(iterate=True,
batch_size=batch_size,
n_steps=n_steps)
states_val, outputs_val, costs_val = theano.function(
[], [states, outputs, costs],
updates=costs.owner.inputs[0].owner.tag.updates)()
assert states_val.shape == (n_steps, batch_size, dim)
assert outputs_val.shape == (n_steps, batch_size)
assert outputs_val.dtype == 'int64'
assert costs_val.shape == (n_steps, batch_size)
def build_model(alphabet_size, config):
layers = config['lstm_layers']
dimensions = [config['lstm_dim_' + str(i)] for i in range(layers)]
uniform_width = config['lstm_init_width']
stack = []
for dim in dimensions:
stack.append(LSTM(dim=dim, use_bias=True,
weights_init = Uniform(width=uniform_width),
forget_init=Constant(1.)))
recurrent_stack = RecurrentStack(stack, name='transition')
readout = Readout(readout_dim=alphabet_size,
source_names=['states#' + str(layers - 1)],
emitter=SoftmaxEmitter(name='emitter'),
feedback_brick=LookupFeedback(alphabet_size,
feedback_dim=alphabet_size,
name='feedback'),
name='readout')
generator = SequenceGenerator(readout=readout,
transition=recurrent_stack,
weights_init=Uniform(width=uniform_width),
biases_init=Constant(0),
name='generator')
generator.push_initialization_config()
generator.initialize()
x = tensor.lmatrix('features')
mask = tensor.fmatrix('features_mask')
cost_matrix = generator.cost_matrix(x, mask=mask)
log2e = math.log(math.e, 2)
if 'batch_length' in config:
length = config['batch_length'] - config['batch_overlap']
cost = log2e * aggregation.mean(cost_matrix[:,-length:].sum(),
mask[:,-length:].sum())
else:
cost = log2e * aggregation.mean(cost_matrix[:,:].sum(),
mask[:,:].sum())
cost.name = 'bits_per_character'
return generator, cost
def test_integer_sequence_generator():
# Disclaimer: here we only check shapes, not values.
readout_dim = 5
feedback_dim = 3
dim = 20
batch_size = 30
n_steps = 10
transition = GatedRecurrent(
name="transition", activation=Tanh(), dim=dim,
weights_init=Orthogonal())
generator = SequenceGenerator(
LinearReadout(readout_dim=readout_dim, source_names=["states"],
emitter=SoftmaxEmitter(name="emitter"),
feedbacker=LookupFeedback(readout_dim, feedback_dim),
name="readout"),
transition,
weights_init=IsotropicGaussian(0.01), biases_init=Constant(0),
name="generator")
generator.initialize()
y = tensor.lmatrix('y')
mask = tensor.matrix('mask')
costs = generator.cost(y, mask)
assert costs.ndim == 2
costs_val = theano.function([y, mask], [costs])(
numpy.zeros((n_steps, batch_size), dtype='int64'),
numpy.ones((n_steps, batch_size), dtype=floatX))[0]
assert costs_val.shape == (n_steps, batch_size)
states, outputs, costs = generator.generate(
iterate=True, batch_size=batch_size, n_steps=n_steps)
states_val, outputs_val, costs_val = theano.function(
[], [states, outputs, costs],
updates=costs.owner.inputs[0].owner.tag.updates)()
assert states_val.shape == (n_steps, batch_size, dim)
assert outputs_val.shape == (n_steps, batch_size)
assert outputs_val.dtype == 'int64'
assert costs_val.shape == (n_steps, batch_size)
def test_recurrentstack_sequence_generator():
"""Test RecurrentStack behaviour inside a SequenceGenerator.
"""
floatX = theano.config.floatX
rng = numpy.random.RandomState(1234)
output_dim = 1
dim = 20
batch_size = 30
n_steps = 10
depth=2
transitions = [LSTM(dim=dim) for _ in range(depth)]
transition = RecurrentStack(transitions,fast=True,
weights_init=Constant(2),
biases_init=Constant(0))
generator = SequenceGenerator(
Readout(readout_dim=output_dim, source_names=["states_%d"%(depth-1)],
emitter=TestEmitter()),
transition,
weights_init=IsotropicGaussian(0.1), biases_init=Constant(0.0),
seed=1234)
generator.initialize()
y = tensor.tensor3('y')
cost = generator.cost(y)
# Check that all states can be accessed and not just the state connected
# to readout.
cg = ComputationGraph(cost)
from blocks.roles import INPUT, OUTPUT
dropout_target = VariableFilter(roles=[INNER_OUTPUT],
# bricks=transitions,
# name_regex='*'
)(cg.variables)
assert_equal(len(dropout_target), depth)
def main():
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
parser = argparse.ArgumentParser(
"Case study of generating a Markov chain with RNN.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"mode",
choices=["train", "sample"],
help="The mode to run. Use `train` to train a new model"
" and `sample` to sample a sequence generated by an"
" existing one.")
parser.add_argument("prefix",
default="sine",
help="The prefix for model, timing and state files")
parser.add_argument("--steps",
type=int,
default=100,
help="Number of steps to plot")
args = parser.parse_args()
dim = 10
num_states = ChainIterator.num_states
feedback_dim = 8
transition = GatedRecurrent(name="transition", activation=Tanh(), dim=dim)
generator = SequenceGenerator(LinearReadout(
readout_dim=num_states,
source_names=["states"],
emitter=SoftmaxEmitter(name="emitter"),
feedbacker=LookupFeedback(num_states, feedback_dim, name='feedback'),
name="readout"),
transition,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
name="generator")
generator.allocate()
logger.debug("Parameters:\n" + pprint.pformat(
[(key, value.get_value().shape)
for key, value in Selector(generator).get_params().items()],
width=120))
if args.mode == "train":
rng = numpy.random.RandomState(1)
batch_size = 50
generator.push_initialization_config()
transition.weights_init = Orthogonal()
generator.initialize()
logger.debug("transition.weights_init={}".format(
transition.weights_init))
cost = generator.cost(tensor.lmatrix('x')).sum()
gh_model = GroundhogModel(generator, cost)
state = GroundhogState(args.prefix, batch_size,
learning_rate=0.0001).as_dict()
data = ChainIterator(rng, 100, batch_size)
trainer = SGD(gh_model, state, data)
main_loop = MainLoop(data, None, None, gh_model, trainer, state, None)
main_loop.main()
elif args.mode == "sample":
load_params(generator, args.prefix + "model.npz")
sample = ComputationGraph(
generator.generate(n_steps=args.steps, batch_size=1,
iterate=True)).function()
states, outputs, costs = [data[:, 0] for data in sample()]
numpy.set_printoptions(precision=3, suppress=True)
print("Generation cost:\n{}".format(costs.sum()))
freqs = numpy.bincount(outputs).astype(floatX)
freqs /= freqs.sum()
print("Frequencies:\n {} vs {}".format(freqs,
ChainIterator.equilibrium))
trans_freqs = numpy.zeros((num_states, num_states), dtype=floatX)
for a, b in zip(outputs, outputs[1:]):
trans_freqs[a, b] += 1
trans_freqs /= trans_freqs.sum(axis=1)[:, None]
print("Transition frequencies:\n{}\nvs\n{}".format(
trans_freqs, ChainIterator.trans_prob))
else:
assert False
def train():
if os.path.isfile('trainingdata.tar'):
with open('trainingdata.tar', 'rb') as f:
main = load(f)
else:
hidden_size = 512
train_dataset = dataset.T_H5PYDataset(
'dataset/wikifonia-seqlen-100.txt.hdf5', which_sets=('train', ))
alphabet_len = train_dataset.vocab_size()
x = theano.tensor.lmatrix('inchar')
recurrent_block = LSTM(dim=hidden_size, activation=Tanh())
recurrent_block2 = LSTM(dim=hidden_size, activation=Tanh())
recurrent_block3 = LSTM(dim=hidden_size, activation=Tanh())
transition = RecurrentStack(
[recurrent_block, recurrent_block2, recurrent_block3])
readout = Readout(readout_dim=alphabet_len,
feedback_brick=LookupFeedback(alphabet_len,
hidden_size,
name='feedback'),
source_names=[
thing for thing in transition.apply.states
if "states" in thing
],
emitter=RandomSoftmaxEmitter(),
name='readout')
gen = SequenceGenerator(readout=readout,
transition=transition,
weights_init=Uniform(width=0.02),
biases_init=Uniform(width=0.0001),
name='sequencegenerator')
gen.push_initialization_config()
gen.initialize()
cost = gen.cost(outputs=x)
cost.name = 'cost'
cg = ComputationGraph(cost)
step_rules = [Adam(), StepClipping(1.0)]
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=CompositeRule(step_rules),
on_unused_sources='ignore')
train_stream = DataStream.default_stream(
train_dataset,
iteration_scheme=SequentialScheme(train_dataset.num_examples,
batch_size=20))
main = MainLoop(model=Model(cost),
data_stream=train_stream,
algorithm=algorithm,
extensions=[
FinishAfter(),
Printing(),
Checkpoint('trainingdata.tar', every_n_epochs=10),
ShowOutput(every_n_epochs=10)
])
main.run()
def test_integer_sequence_generator():
"""Test a sequence generator with integer outputs.
Such sequence generators can be used to e.g. model language.
"""
rng = numpy.random.RandomState(1234)
readout_dim = 5
feedback_dim = 3
dim = 20
batch_size = 30
n_steps = 10
transition = GatedRecurrent(dim=dim, activation=Tanh(),
weights_init=Orthogonal())
generator = SequenceGenerator(
Readout(readout_dim=readout_dim, source_names=["states"],
emitter=SoftmaxEmitter(theano_seed=1234),
feedback_brick=LookupFeedback(readout_dim,
feedback_dim)),
transition,
weights_init=IsotropicGaussian(0.1), biases_init=Constant(0),
seed=1234)
generator.initialize()
# Test 'cost_matrix' method
y = tensor.lmatrix('y')
mask = tensor.matrix('mask')
costs = generator.cost_matrix(y, mask)
assert costs.ndim == 2
costs_fun = theano.function([y, mask], [costs])
y_test = rng.randint(readout_dim, size=(n_steps, batch_size))
m_test = numpy.ones((n_steps, batch_size), dtype=floatX)
costs_val = costs_fun(y_test, m_test)[0]
assert costs_val.shape == (n_steps, batch_size)
assert_allclose(costs_val.sum(), 482.827, rtol=1e-5)
# Test 'cost' method
cost = generator.cost(y, mask)
assert cost.ndim == 0
cost_val = theano.function([y, mask], [cost])(y_test, m_test)
assert_allclose(cost_val, 16.0942, rtol=1e-5)
# Test 'AUXILIARY' variable 'per_sequence_element' in 'cost' method
cg = ComputationGraph([cost])
var_filter = VariableFilter(roles=[AUXILIARY])
aux_var_name = '_'.join([generator.name, generator.cost.name,
'per_sequence_element'])
cost_per_el = [el for el in var_filter(cg.variables)
if el.name == aux_var_name][0]
assert cost_per_el.ndim == 0
cost_per_el_val = theano.function([y, mask], [cost_per_el])(y_test, m_test)
assert_allclose(cost_per_el_val, 1.60942, rtol=1e-5)
# Test generate
states, outputs, costs = generator.generate(
iterate=True, batch_size=batch_size, n_steps=n_steps)
cg = ComputationGraph(states + outputs + costs)
states_val, outputs_val, costs_val = theano.function(
[], [states, outputs, costs],
updates=cg.updates)()
assert states_val.shape == (n_steps, batch_size, dim)
assert outputs_val.shape == (n_steps, batch_size)
assert outputs_val.dtype == 'int64'
assert costs_val.shape == (n_steps, batch_size)
assert_allclose(states_val.sum(), -17.91811, rtol=1e-5)
assert_allclose(costs_val.sum(), 482.863, rtol=1e-5)
assert outputs_val.sum() == 630
# Test masks agnostic results of cost
cost1 = costs_fun([[1], [2]], [[1], [1]])[0]
cost2 = costs_fun([[3, 1], [4, 2], [2, 0]],
[[1, 1], [1, 1], [1, 0]])[0]
assert_allclose(cost1.sum(), cost2[:, 1].sum(), rtol=1e-5)
def test_with_attention():
"""Test a sequence generator with continuous outputs and attention."""
rng = numpy.random.RandomState(1234)
inp_dim = 2
inp_len = 10
attended_dim = 3
attended_len = 11
batch_size = 4
n_steps = 30
# For values
def rand(size):
return rng.uniform(size=size).astype(floatX)
# For masks
def generate_mask(length, batch_size):
mask = numpy.ones((length, batch_size), dtype=floatX)
# To make it look like read data
for i in range(batch_size):
mask[1 + rng.randint(0, length - 1):, i] = 0.0
return mask
output_vals = rand((inp_len, batch_size, inp_dim))
output_mask_vals = generate_mask(inp_len, batch_size)
attended_vals = rand((attended_len, batch_size, attended_dim))
attended_mask_vals = generate_mask(attended_len, batch_size)
transition = TestTransition(
dim=inp_dim, attended_dim=attended_dim, activation=Identity())
attention = SequenceContentAttention(
state_names=transition.apply.states, match_dim=inp_dim)
generator = SequenceGenerator(
Readout(
readout_dim=inp_dim,
source_names=[transition.apply.states[0],
attention.take_glimpses.outputs[0]],
emitter=TestEmitter()),
transition=transition,
attention=attention,
weights_init=IsotropicGaussian(0.1), biases_init=Constant(0),
add_contexts=False, seed=1234)
generator.initialize()
# Test 'cost_matrix' method
attended = tensor.tensor3("attended")
attended_mask = tensor.matrix("attended_mask")
outputs = tensor.tensor3('outputs')
mask = tensor.matrix('mask')
costs = generator.cost_matrix(outputs, mask,
attended=attended,
attended_mask=attended_mask)
costs_vals = costs.eval({outputs: output_vals,
mask: output_mask_vals,
attended: attended_vals,
attended_mask: attended_mask_vals})
assert costs_vals.shape == (inp_len, batch_size)
assert_allclose(costs_vals.sum(), 13.5042, rtol=1e-5)
# Test `generate` method
results = (
generator.generate(n_steps=n_steps, batch_size=attended.shape[1],
attended=attended, attended_mask=attended_mask))
assert len(results) == 5
states_vals, outputs_vals, glimpses_vals, weights_vals, costs_vals = (
theano.function([attended, attended_mask], results)
(attended_vals, attended_mask_vals))
assert states_vals.shape == (n_steps, batch_size, inp_dim)
assert states_vals.shape == outputs_vals.shape
assert glimpses_vals.shape == (n_steps, batch_size, attended_dim)
assert weights_vals.shape == (n_steps, batch_size, attended_len)
assert costs_vals.shape == (n_steps, batch_size)
assert_allclose(states_vals.sum(), 23.4172, rtol=1e-5)
# There is no generation cost in this case, since generation is
# deterministic
assert_allclose(costs_vals.sum(), 0.0, rtol=1e-5)
assert_allclose(weights_vals.sum(), 120.0, rtol=1e-5)
assert_allclose(glimpses_vals.sum(), 199.2402, rtol=1e-5)
assert_allclose(outputs_vals.sum(), -11.6008, rtol=1e-5)
def main():
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
parser = argparse.ArgumentParser(
"Case study of generating a Markov chain with RNN.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"mode", choices=["train", "sample"],
help="The mode to run. Use `train` to train a new model"
" and `sample` to sample a sequence generated by an"
" existing one.")
parser.add_argument(
"prefix", default="sine",
help="The prefix for model, timing and state files")
parser.add_argument(
"--steps", type=int, default=100,
help="Number of steps to plot")
args = parser.parse_args()
dim = 10
num_states = ChainIterator.num_states
feedback_dim = 8
transition = GatedRecurrent(name="transition", activation=Tanh(), dim=dim)
generator = SequenceGenerator(
LinearReadout(readout_dim=num_states, source_names=["states"],
emitter=SoftmaxEmitter(name="emitter"),
feedbacker=LookupFeedback(
num_states, feedback_dim, name='feedback'),
name="readout"),
transition,
weights_init=IsotropicGaussian(0.01), biases_init=Constant(0),
name="generator")
generator.allocate()
logger.debug("Parameters:\n" +
pprint.pformat(
[(key, value.get_value().shape) for key, value
in Selector(generator).get_params().items()],
width=120))
if args.mode == "train":
rng = numpy.random.RandomState(1)
batch_size = 50
generator.push_initialization_config()
transition.weights_init = Orthogonal()
generator.initialize()
logger.debug("transition.weights_init={}".format(
transition.weights_init))
cost = generator.cost(tensor.lmatrix('x')).sum()
gh_model = GroundhogModel(generator, cost)
state = GroundhogState(args.prefix, batch_size,
learning_rate=0.0001).as_dict()
data = ChainIterator(rng, 100, batch_size)
trainer = SGD(gh_model, state, data)
main_loop = MainLoop(data, None, None, gh_model, trainer, state, None)
main_loop.main()
elif args.mode == "sample":
load_params(generator, args.prefix + "model.npz")
sample = ComputationGraph(generator.generate(
n_steps=args.steps, batch_size=1, iterate=True)).function()
states, outputs, costs = [data[:, 0] for data in sample()]
numpy.set_printoptions(precision=3, suppress=True)
print("Generation cost:\n{}".format(costs.sum()))
freqs = numpy.bincount(outputs).astype(floatX)
freqs /= freqs.sum()
print("Frequencies:\n {} vs {}".format(freqs,
ChainIterator.equilibrium))
trans_freqs = numpy.zeros((num_states, num_states), dtype=floatX)
for a, b in zip(outputs, outputs[1:]):
trans_freqs[a, b] += 1
trans_freqs /= trans_freqs.sum(axis=1)[:, None]
print("Transition frequencies:\n{}\nvs\n{}".format(
trans_freqs, ChainIterator.trans_prob))
else:
assert False
def train():
if os.path.isfile('trainingdata.tar'):
with open('trainingdata.tar', 'rb') as f:
main = load(f)
else:
hidden_size = 512
filename = 'warpeace.hdf5'
encoder = HDF5CharEncoder('warpeace_input.txt', 1000)
encoder.write(filename)
alphabet_len = encoder.length
x = theano.tensor.lmatrix('x')
readout = Readout(
readout_dim=alphabet_len,
feedback_brick=LookupFeedback(alphabet_len, hidden_size, name='feedback'),
source_names=['states'],
emitter=RandomSoftmaxEmitter(),
name='readout'
)
transition = GatedRecurrent(
activation=Tanh(),
dim=hidden_size)
transition.weights_init = IsotropicGaussian(0.01)
gen = SequenceGenerator(readout=readout,
transition=transition,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
name='sequencegenerator')
gen.push_initialization_config()
gen.initialize()
cost = gen.cost(outputs=x)
cost.name = 'cost'
cg = ComputationGraph(cost)
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Scale(0.5))
train_set = encoder.get_dataset()
train_stream = DataStream.default_stream(
train_set, iteration_scheme=SequentialScheme(
train_set.num_examples, batch_size=128))
main = MainLoop(
model=Model(cost),
data_stream=train_stream,
algorithm=algorithm,
extensions=[
FinishAfter(),
Printing(),
Checkpoint('trainingdata.tar', every_n_epochs=10),
ShowOutput(every_n_epochs=10)
])
main.run()
def test_sequence_generator_with_lm():
floatX = theano.config.floatX
rng = numpy.random.RandomState(1234)
readout_dim = 5
feedback_dim = 3
dim = 20
batch_size = 30
n_steps = 10
transition = GatedRecurrent(dim=dim, activation=Tanh(),
weights_init=Orthogonal())
language_model = SequenceGenerator(
Readout(readout_dim=readout_dim, source_names=["states"],
emitter=SoftmaxEmitter(theano_seed=1234),
feedback_brick=LookupFeedback(readout_dim, dim,
name='feedback')),
SimpleRecurrent(dim, Tanh()),
name='language_model')
generator = SequenceGenerator(
Readout(readout_dim=readout_dim, source_names=["states", "lm_states"],
emitter=SoftmaxEmitter(theano_seed=1234),
feedback_brick=LookupFeedback(readout_dim,
feedback_dim)),
transition,
language_model=language_model,
weights_init=IsotropicGaussian(0.1), biases_init=Constant(0),
seed=1234)
generator.initialize()
# Test 'cost_matrix' method
y = tensor.lmatrix('y')
y.tag.test_value = numpy.zeros((15, batch_size), dtype='int64')
mask = tensor.matrix('mask')
mask.tag.test_value = numpy.ones((15, batch_size))
costs = generator.cost_matrix(y, mask)
assert costs.ndim == 2
costs_fun = theano.function([y, mask], [costs])
y_test = rng.randint(readout_dim, size=(n_steps, batch_size))
m_test = numpy.ones((n_steps, batch_size), dtype=floatX)
costs_val = costs_fun(y_test, m_test)[0]
assert costs_val.shape == (n_steps, batch_size)
assert_allclose(costs_val.sum(), 483.153, rtol=1e-5)
# Test 'cost' method
cost = generator.cost(y, mask)
assert cost.ndim == 0
cost_val = theano.function([y, mask], cost)(y_test, m_test)
assert_allclose(cost_val, 16.105, rtol=1e-5)
# Test 'AUXILIARY' variable 'per_sequence_element' in 'cost' method
cg = ComputationGraph([cost])
var_filter = VariableFilter(roles=[AUXILIARY])
aux_var_name = '_'.join([generator.name, generator.cost.name,
'per_sequence_element'])
cost_per_el = [el for el in var_filter(cg.variables)
if el.name == aux_var_name][0]
assert cost_per_el.ndim == 0
cost_per_el_val = theano.function([y, mask], [cost_per_el])(y_test, m_test)
assert_allclose(cost_per_el_val, 1.61051, rtol=1e-5)
# Test generate
states, outputs, lm_states, costs = generator.generate(
iterate=True, batch_size=batch_size, n_steps=n_steps)
cg = ComputationGraph([states, outputs, costs])
states_val, outputs_val, costs_val = theano.function(
[], [states, outputs, costs],
updates=cg.updates)()
assert states_val.shape == (n_steps, batch_size, dim)
assert outputs_val.shape == (n_steps, batch_size)
assert outputs_val.dtype == 'int64'
assert costs_val.shape == (n_steps, batch_size)
assert_allclose(states_val.sum(), -4.88367, rtol=1e-5)
assert_allclose(costs_val.sum(), 486.681, rtol=1e-5)
assert outputs_val.sum() == 627
# Test masks agnostic results of cost
cost1 = costs_fun([[1], [2]], [[1], [1]])[0]
cost2 = costs_fun([[3, 1], [4, 2], [2, 0]],
[[1, 1], [1, 1], [1, 0]])[0]
assert_allclose(cost1.sum(), cost2[:, 1].sum(), rtol=1e-5)
def main():
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
parser = argparse.ArgumentParser(
"Case study of language modeling with RNN",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"mode",
choices=["train", "sample"],
help="The mode to run. Use `train` to train a new model"
" and `sample` to sample a sequence generated by an"
" existing one.")
parser.add_argument("prefix",
default="sine",
help="The prefix for model, timing and state files")
parser.add_argument("state",
nargs="?",
default="",
help="Changes to Groundhog state")
parser.add_argument("--path", help="Path to a language dataset")
parser.add_argument("--dict", help="Path to the dataset dictionary")
parser.add_argument("--restart", help="Start anew")
parser.add_argument("--reset",
action="store_true",
default=False,
help="Reset the hidden state between batches")
parser.add_argument("--steps",
type=int,
default=100,
help="Number of steps to plot for the 'sample' mode"
" OR training sequence length for the 'train' mode.")
args = parser.parse_args()
logger.debug("Args:\n" + str(args))
dim = 200
num_chars = 50
transition = GatedRecurrent(name="transition",
activation=Tanh(),
dim=dim,
weights_init=Orthogonal())
generator = SequenceGenerator(LinearReadout(
readout_dim=num_chars,
source_names=["states"],
emitter=SoftmaxEmitter(name="emitter"),
feedbacker=LookupFeedback(num_chars, dim, name='feedback'),
name="readout"),
transition,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
name="generator")
generator.allocate()
logger.debug("Parameters:\n" + pprint.pformat(
[(key, value.get_value().shape)
for key, value in Selector(generator).get_params().items()],
width=120))
if args.mode == "train":
batch_size = 1
seq_len = args.steps
generator.initialize()
# Build cost computation graph that uses the saved hidden states.
# An issue: for Groundhog this is completely transparent, that's
# why it does not carry the hidden state over the period when
# validation in done. We should find a way to fix in the future.
x = tensor.lmatrix('x')
init_states = shared_floatx_zeros((batch_size, dim),
name='init_states')
reset = tensor.scalar('reset')
cost = ComputationGraph(
generator.cost(x, states=init_states * reset).sum())
# TODO: better search routine
states = [
v for v in cost.variables if hasattr(v.tag, 'application_call')
and v.tag.application_call.brick == generator.transition and
(v.tag.application_call.application == generator.transition.apply)
and v.tag.role == VariableRole.OUTPUT and v.tag.name == 'states'
]
assert len(states) == 1
states = states[0]
gh_model = GroundhogModel(generator, cost)
gh_model.properties.append(
('bpc', cost.outputs[0] * numpy.log(2) / seq_len))
gh_model.properties.append(('mean_init_state', init_states.mean()))
gh_model.properties.append(('reset', reset))
if not args.reset:
gh_model.updates.append((init_states, states[-1]))
state = GroundhogState(args.prefix, batch_size,
learning_rate=0.0001).as_dict()
changes = eval("dict({})".format(args.state))
state.update(changes)
def output_format(x, y, reset):
return dict(x=x[:, None], reset=reset)
train, valid, test = [
LMIterator(batch_size=batch_size,
use_infinite_loop=mode == 'train',
path=args.path,
seq_len=seq_len,
mode=mode,
chunks='chars',
output_format=output_format,
can_fit=True) for mode in ['train', 'valid', 'test']
]
trainer = SGD(gh_model, state, train)
state['on_nan'] = 'warn'
state['cutoff'] = 1.
main_loop = MainLoop(train, valid, None, gh_model, trainer, state,
None)
if not args.restart:
main_loop.load()
main_loop.main()
elif args.mode == "sample":
load_params(generator, args.prefix + "model.npz")
chars = numpy.load(args.dict)['unique_chars']
sample = ComputationGraph(
generator.generate(n_steps=args.steps, batch_size=10,
iterate=True)).function()
states, outputs, costs = sample()
for i in range(10):
print("Generation cost: {}".format(costs[:, i].sum()))
print("".join([chars[o] for o in outputs[:, i]]))
else:
assert False
def main_rnn(config):
x = tensor.tensor3('features')
y = tensor.matrix('targets')
# if 'LSTM' in config['model'] :
# from models import getLSTMstack
# y_hat = getLSTMstack(input_dim=13, input_var=x, depth=int(config['model'][-1]))
# else :
# raise Exception("These are not the LSTM we are looking for")
# y_hat = model.apply(x)
emitter = TestEmitter()
# emitter = TrivialEmitter(readout_dim=config['lstm_hidden_size'])
# cost_func = SquaredError()
# @application
# def qwe(self, readouts, outputs=None):
# print(type(self), type(readouts))
# x = cost_func.apply(readouts,outputs)
# return x
print(type(emitter.cost))
# emitter.cost = qwe
# print(type(qwe))
steps = 2
n_samples= config['target_size']
transition = [LSTM(config['lstm_hidden_size']) for _ in range(4)]
transition = RecurrentStack(transition,
name="transition", skip_connections=False)
source_names = [name for name in transition.apply.states if 'states' in name]
readout = Readout(emitter, readout_dim=config['lstm_hidden_size'], source_names=source_names,feedback_brick=None, merge=None, merge_prototype=None, post_merge=None, merged_dim=None)
seqgen = SequenceGenerator(readout, transition, attention=None, add_contexts=False)
seqgen.weights_init = IsotropicGaussian(0.01)
seqgen.biases_init = Constant(0.)
seqgen.push_initialization_config()
seqgen.transition.biases_init = IsotropicGaussian(0.01,1)
seqgen.transition.push_initialization_config()
seqgen.initialize()
states = seqgen.transition.apply.outputs
print('states',states)
states = {name: shared_floatx_zeros((n_samples, config['lstm_hidden_size']))
for name in states}
cost_matrix = seqgen.cost_matrix(x, **states)
cost = cost_matrix.mean()
cost.name = "nll"
cg = ComputationGraph(cost)
model = Model(cost)
#Cost
# cost = SquaredError().apply(y_hat ,y)
#cost = CategoricalCrossEntropy().apply(T.flatten(),Y)
#
#for sampling
#cg = ComputationGraph(seqgen.generate(n_steps=steps,batch_size=n_samples, iterate=True))
algorithm = GradientDescent(
cost=cost, parameters=cg.parameters,
step_rule=Scale(learning_rate=config['learning_rate']))
#Getting the stream
train_stream = MFCC.get_stream(config['batch_size'],config['source_size'],config['target_size'],config['num_examples'])
#Monitoring stuff
extensions = [Timing(),
FinishAfter(after_n_batches=config['num_batches']),
#DataStreamMonitoring([cost, error_rate],test_stream,prefix="test"),
TrainingDataMonitoring([cost], prefix="train", every_n_batches=1),
#Checkpoint(save_to),
ProgressBar(),
Printing(every_n_batches=1)]
main_loop = MainLoop(
algorithm,
train_stream,
# model=model,
extensions=extensions)
main_loop.run()
def main(config):
vocab_src, _ = text_to_dict([config['train_src'],
config['dev_src'], config['test_src']])
vocab_tgt, cabvo = text_to_dict([config['train_tgt'],
config['dev_tgt']])
# Create Theano variables
logger.info('Creating theano variables')
source_sentence = tensor.lmatrix('source')
source_sentence_mask = tensor.matrix('source_mask')
target_sentence = tensor.lmatrix('target')
target_sentence_mask = tensor.matrix('target_mask')
source_sentence.tag.test_value = [[13, 20, 0, 20, 0, 20, 0],
[1, 4, 8, 4, 8, 4, 8],]
source_sentence_mask.tag.test_value = [[0, 1, 0, 1, 0, 1, 0],
[1, 0, 1, 0, 1, 0, 1],]
target_sentence.tag.test_value = [[0,1,1,5],
[2,0,1,0],]
target_sentence_mask.tag.test_value = [[0,1,1,0],
[1,1,1,0],]
logger.info('Building RNN encoder-decoder')
### Building Encoder
embedder = LookupTable(
length=len(vocab_src),
dim=config['embed_src'],
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0),
name='embedder')
transformer = Linear(
config['embed_src'],
config['hidden_src']*4,
weights_init=IsotropicGaussian(),
biases_init=Constant(0.0),
name='transformer')
lstminit = np.asarray([0.0,]*config['hidden_src']+[0.0,]*config['hidden_src']+[1.0,]*config['hidden_src']+[0.0,]*config['hidden_src'])
encoder = Bidirectional(
LSTM(
dim=config['hidden_src'],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(lstminit)),
name='encoderBiLSTM'
)
encoder.prototype.weights_init = Orthogonal()
### Building Decoder
lstminit = np.asarray([0.0,]*config['hidden_tgt']+[0.0,]*config['hidden_tgt']+[1.0,]*config['hidden_tgt']+[0.0,]*config['hidden_tgt'])
transition = LSTM2GO(
attended_dim=config['hidden_tgt'],
dim=config['hidden_tgt'],
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(lstminit),
name='decoderLSTM')
attention = SequenceContentAttention(
state_names=transition.apply.states, # default activation is Tanh
state_dims=[config['hidden_tgt']],
attended_dim=config['hidden_src']*2,
match_dim=config['hidden_tgt'],
name="attention")
readout = Readout(
source_names=['states',
'feedback',
attention.take_glimpses.outputs[0]],
readout_dim=len(vocab_tgt),
emitter = SoftmaxEmitter(
name='emitter'),
feedback_brick = LookupFeedback(
num_outputs=len(vocab_tgt),
feedback_dim=config['embed_tgt'],
name='feedback'),
post_merge=InitializableFeedforwardSequence([
Bias(dim=config['hidden_tgt'],
name='softmax_bias').apply,
Linear(input_dim=config['hidden_tgt'],
output_dim=config['embed_tgt'],
use_bias=False,
name='softmax0').apply,
Linear(input_dim=config['embed_tgt'],
name='softmax1').apply]),
merged_dim=config['hidden_tgt'])
decoder = SequenceGenerator(
readout=readout,
transition=transition,
attention=attention,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
name="generator",
fork=Fork(
[name for name in transition.apply.sequences if name != 'mask'],
prototype=Linear()),
add_contexts=True)
decoder.transition.weights_init = Orthogonal()
#printchildren(encoder, 1)
# Initialize model
logger.info('Initializing model')
embedder.initialize()
transformer.initialize()
encoder.initialize()
decoder.initialize()
# Apply model
embedded = embedder.apply(source_sentence)
tansformed = transformer.apply(embedded)
encoded = encoder.apply(tansformed)[0]
generated = decoder.generate(
n_steps=2*source_sentence.shape[1],
batch_size=source_sentence.shape[0],
attended = encoded.dimshuffle(1,0,2),
attended_mask=tensor.ones(source_sentence.shape).T
)
print 'Generated: ', generated
# generator_generate_outputs
#samples = generated[1] # For GRU
samples = generated[2] # For LSTM
samples.name = 'samples'
#samples_cost = generated[4] # For GRU
samples_cost = generated[5] # For LSTM
samples_cost = 'sampling_cost'
cost = decoder.cost(
mask = target_sentence_mask.T,
outputs = target_sentence.T,
attended = encoded.dimshuffle(1,0,2),
attended_mask = source_sentence_mask.T)
cost.name = 'target_cost'
cost.tag.aggregation_scheme = TakeLast(cost)
model = Model(cost)
logger.info('Creating computational graph')
cg = ComputationGraph(cost)
# apply dropout for regularization
if config['dropout'] < 1.0: # dropout is applied to the output of maxout in ghog
logger.info('Applying dropout')
dropout_inputs = [x for x in cg.intermediary_variables if x.name == 'maxout_apply_output']
cg = apply_dropout(cg, dropout_inputs, config['dropout'])
########
# Print shapes
shapes = [param.get_value().shape for param in cg.parameters]
logger.info("Parameter shapes: ")
for shape, count in Counter(shapes).most_common():
logger.info(' {:15}: {}'.format(shape, count))
logger.info("Total number of parameters: {}".format(len(shapes)))
printchildren(embedder, 1)
printchildren(transformer, 1)
printchildren(encoder, 1)
printchildren(decoder, 1)
# Print parameter names
# enc_dec_param_dict = merge(Selector(embedder).get_parameters(), Selector(encoder).get_parameters(), Selector(decoder).get_parameters())
# enc_dec_param_dict = merge(Selector(decoder).get_parameters())
# logger.info("Parameter names: ")
# for name, value in enc_dec_param_dict.items():
# logger.info(' {:15}: {}'.format(value.get_value().shape, name))
# logger.info("Total number of parameters: {}".format(len(enc_dec_param_dict)))
##########
# Training data
train_stream = get_train_stream(config,
[config['train_src'],], [config['train_tgt'],],
vocab_src, vocab_tgt)
dev_stream = get_dev_stream(
[config['dev_src'],], [config['dev_tgt'],],
vocab_src, vocab_tgt)
test_stream = get_test_stream([config['test_src'],], vocab_src)
# Set extensions
logger.info("Initializing extensions")
extensions = [
FinishAfter(after_n_batches=config['finish_after']),
ProgressBar(),
TrainingDataMonitoring([cost],
prefix="tra",
after_batch=True),
DataStreamMonitoring(variables=[cost],
data_stream=dev_stream,
prefix="dev",
after_batch=True),
Sampler(
model=Model(samples),
data_stream=dev_stream,
vocab=cabvo,
saveto=config['saveto']+'dev',
every_n_batches=config['save_freq']),
Sampler(
model=Model(samples),
data_stream=test_stream,
vocab=cabvo,
saveto=config['saveto']+'test',
after_n_batches=1,
on_resumption=True,
before_training=True),
Plotter(saveto=config['saveto'], after_batch=True),
Printing(after_batch=True),
Checkpoint(
path=config['saveto'],
parameters = cg.parameters,
save_main_loop=False,
every_n_batches=config['save_freq'])]
if BOKEH_AVAILABLE:
Plot('Training cost', channels=[['target_cost']], after_batch=True)
if config['reload']:
extensions.append(Load(path=config['saveto'],
load_iteration_state=False,
load_log=False))
else:
with open(config['saveto']+'.txt', 'w') as f:
pass
# Set up training algorithm
logger.info("Initializing training algorithm")
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=CompositeRule([StepClipping(config['step_clipping']),
eval(config['step_rule'])()])
)
# Initialize main loop
logger.info("Initializing main loop")
main_loop = MainLoop(
model=model,
algorithm=algorithm,
data_stream=train_stream,
extensions=extensions)
main_loop.run()
def main():
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
parser = argparse.ArgumentParser(
"Case study of generating a Markov chain with RNN.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"mode", choices=["train", "sample"],
help="The mode to run. Use `train` to train a new model"
" and `sample` to sample a sequence generated by an"
" existing one.")
parser.add_argument(
"save_path", default="sine",
help="The part to save PyLearn2 model")
parser.add_argument(
"--steps", type=int, default=100,
help="Number of steps to plot")
parser.add_argument(
"--reset", action="store_true", default=False,
help="Start training from scratch")
args = parser.parse_args()
num_states = ChainDataset.num_states
if args.mode == "train":
# Experiment configuration
rng = numpy.random.RandomState(1)
batch_size = 50
seq_len = 100
dim = 10
feedback_dim = 8
# Build the bricks and initialize them
transition = GatedRecurrent(name="transition", activation=Tanh(),
dim=dim)
generator = SequenceGenerator(
LinearReadout(readout_dim=num_states, source_names=["states"],
emitter=SoftmaxEmitter(name="emitter"),
feedbacker=LookupFeedback(
num_states, feedback_dim, name='feedback'),
name="readout"),
transition,
weights_init=IsotropicGaussian(0.01), biases_init=Constant(0),
name="generator")
generator.push_initialization_config()
transition.weights_init = Orthogonal()
generator.initialize()
logger.debug("Parameters:\n" +
pprint.pformat(
[(key, value.get_value().shape) for key, value
in Selector(generator).get_params().items()],
width=120))
logger.debug("Markov chain entropy: {}".format(
ChainDataset.entropy))
logger.debug("Expected min error: {}".format(
-ChainDataset.entropy * seq_len * batch_size))
if os.path.isfile(args.save_path) and not args.reset:
model = Pylearn2Model.load(args.save_path)
else:
model = Pylearn2Model(generator)
# Build the cost computation graph.
# Note: would be probably nicer to make cost part of the model.
x = tensor.ltensor3('x')
cost = Pylearn2Cost(model.brick.cost(x[:, :, 0]).sum())
dataset = ChainDataset(rng, seq_len)
sgd = SGD(learning_rate=0.0001, cost=cost,
batch_size=batch_size, batches_per_iter=10,
monitoring_dataset=dataset,
monitoring_batch_size=batch_size,
monitoring_batches=1,
learning_rule=Pylearn2LearningRule(
SGDLearningRule(),
dict(training_objective=cost.cost)))
train = Pylearn2Train(dataset, model, algorithm=sgd,
save_path=args.save_path, save_freq=10)
train.main_loop()
elif args.mode == "sample":
model = Pylearn2Model.load(args.save_path)
generator = model.brick
sample = ComputationGraph(generator.generate(
n_steps=args.steps, batch_size=1, iterate=True)).function()
states, outputs, costs = [data[:, 0] for data in sample()]
numpy.set_printoptions(precision=3, suppress=True)
print("Generation cost:\n{}".format(costs.sum()))
freqs = numpy.bincount(outputs).astype(floatX)
freqs /= freqs.sum()
print("Frequencies:\n {} vs {}".format(freqs,
ChainDataset.equilibrium))
trans_freqs = numpy.zeros((num_states, num_states), dtype=floatX)
for a, b in zip(outputs, outputs[1:]):
trans_freqs[a, b] += 1
trans_freqs /= trans_freqs.sum(axis=1)[:, None]
print("Transition frequencies:\n{}\nvs\n{}".format(
trans_freqs, ChainDataset.trans_prob))
else:
assert False
source_names=["states#2"],
emitter=SoftmaxEmitter(name="emitter"),
feedback_brick=LookupFeedback(alphabet_size,
feedback_dim=alphabet_size,
name="feedback"),
name="readout")
seq_gen = SequenceGenerator(readout=readout,
transition=rnn,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
name="generator")
seq_gen.push_initialization_config()
rnn.weights_init = Orthogonal()
seq_gen.initialize()
# z markov_tutorial
x = tensor.lvector('features')
x = x.reshape( (x.shape[0], 1) )
cost = aggregation.mean(seq_gen.cost_matrix(x[:,:]).sum(), x.shape[1])
cost.name = "sequence_log_likelihood"
cost_cg = ComputationGraph(cost)
# theano.printing.pydotprint(cost, outfile="./pics/symbolic_graph_unopt.png", var_with_name_simple=True)
algorithm = GradientDescent(
cost=cost,
parameters=list(Selector(seq_gen).get_parameters().values()),
step_rule=Scale(0.001))
def main(mode, save_path, num_batches, from_dump):
if mode == "train":
# Experiment configuration
dimension = 100
readout_dimension = len(char2code)
# Data processing pipeline
data_stream = DataStreamMapping(
mapping=lambda data: tuple(array.T for array in data),
data_stream=PaddingDataStream(
BatchDataStream(
iteration_scheme=ConstantScheme(10),
data_stream=DataStreamMapping(
mapping=reverse_words,
add_sources=("targets", ),
data_stream=DataStreamFilter(
predicate=lambda data: len(data[0]) <= 100,
data_stream=OneBillionWord(
"training", [99],
char2code,
level="character",
preprocess=str.lower).get_default_stream())))))
# Build the model
chars = tensor.lmatrix("features")
chars_mask = tensor.matrix("features_mask")
targets = tensor.lmatrix("targets")
targets_mask = tensor.matrix("targets_mask")
encoder = Bidirectional(GatedRecurrent(dim=dimension,
activation=Tanh()),
weights_init=Orthogonal())
encoder.initialize()
fork = Fork([
name
for name in encoder.prototype.apply.sequences if name != 'mask'
],
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0))
fork.input_dim = dimension
fork.fork_dims = {name: dimension for name in fork.fork_names}
fork.initialize()
lookup = LookupTable(readout_dimension,
dimension,
weights_init=IsotropicGaussian(0.1))
lookup.initialize()
transition = Transition(activation=Tanh(),
dim=dimension,
attended_dim=2 * dimension,
name="transition")
attention = SequenceContentAttention(
state_names=transition.apply.states,
match_dim=dimension,
name="attention")
readout = LinearReadout(readout_dim=readout_dimension,
source_names=["states"],
emitter=SoftmaxEmitter(name="emitter"),
feedbacker=LookupFeedback(
readout_dimension, dimension),
name="readout")
generator = SequenceGenerator(readout=readout,
transition=transition,
attention=attention,
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0),
name="generator")
generator.push_initialization_config()
transition.weights_init = Orthogonal()
generator.initialize()
bricks = [encoder, fork, lookup, generator]
# Give an idea of what's going on
params = Selector(bricks).get_params()
logger.info("Parameters:\n" +
pprint.pformat([(key, value.get_value().shape)
for key, value in params.items()],
width=120))
# Build the cost computation graph
batch_cost = generator.cost(
targets,
targets_mask,
attended=encoder.apply(**dict_union(fork.apply(
lookup.lookup(chars), return_dict=True),
mask=chars_mask)),
attended_mask=chars_mask).sum()
batch_size = named_copy(chars.shape[1], "batch_size")
cost = aggregation.mean(batch_cost, batch_size)
cost.name = "sequence_log_likelihood"
logger.info("Cost graph is built")
# Fetch variables useful for debugging
max_length = named_copy(chars.shape[0], "max_length")
cost_per_character = named_copy(
aggregation.mean(batch_cost, batch_size * max_length),
"character_log_likelihood")
cg = ComputationGraph(cost)
energies = unpack(VariableFilter(application=readout.readout,
name="output")(cg.variables),
singleton=True)
min_energy = named_copy(energies.min(), "min_energy")
max_energy = named_copy(energies.max(), "max_energy")
(activations, ) = VariableFilter(
application=generator.transition.apply,
name="states")(cg.variables)
mean_activation = named_copy(activations.mean(), "mean_activation")
# Define the training algorithm.
algorithm = GradientDescent(cost=cost,
step_rule=CompositeRule([
GradientClipping(10.0),
SteepestDescent(0.01)
]))
observables = [
cost, min_energy, max_energy, mean_activation, batch_size,
max_length, cost_per_character, algorithm.total_step_norm,
algorithm.total_gradient_norm
]
for name, param in params.items():
observables.append(named_copy(param.norm(2), name + "_norm"))
observables.append(
named_copy(algorithm.gradients[param].norm(2),
name + "_grad_norm"))
main_loop = MainLoop(
model=bricks,
data_stream=data_stream,
algorithm=algorithm,
extensions=([LoadFromDump(from_dump)] if from_dump else []) + [
Timing(),
TrainingDataMonitoring(observables, after_every_batch=True),
TrainingDataMonitoring(
observables, prefix="average", every_n_batches=10),
FinishAfter(after_n_batches=num_batches).add_condition(
"after_batch", lambda log: math.isnan(
log.current_row.total_gradient_norm)),
Plot(os.path.basename(save_path),
[["average_" + cost.name],
["average_" + cost_per_character.name]],
every_n_batches=10),
SerializeMainLoop(save_path,
every_n_batches=500,
save_separately=["model", "log"]),
Printing(every_n_batches=1)
])
main_loop.run()
elif mode == "test":
with open(save_path, "rb") as source:
encoder, fork, lookup, generator = dill.load(source)
logger.info("Model is loaded")
chars = tensor.lmatrix("features")
generated = generator.generate(
n_steps=3 * chars.shape[0],
batch_size=chars.shape[1],
attended=encoder.apply(**dict_union(
fork.apply(lookup.lookup(chars), return_dict=True))),
attended_mask=tensor.ones(chars.shape))
sample_function = ComputationGraph(generated).get_theano_function()
logging.info("Sampling function is compiled")
while True:
# Python 2-3 compatibility
line = input("Enter a sentence\n")
batch_size = int(input("Enter a number of samples\n"))
encoded_input = [
char2code.get(char, char2code["<UNK>"])
for char in line.lower().strip()
]
encoded_input = ([char2code['<S>']] + encoded_input +
[char2code['</S>']])
print("Encoder input:", encoded_input)
target = reverse_words((encoded_input, ))[0]
print("Target: ", target)
states, samples, glimpses, weights, costs = sample_function(
numpy.repeat(numpy.array(encoded_input)[:, None],
batch_size,
axis=1))
messages = []
for i in range(samples.shape[1]):
sample = list(samples[:, i])
try:
true_length = sample.index(char2code['</S>']) + 1
except ValueError:
true_length = len(sample)
sample = sample[:true_length]
cost = costs[:true_length, i].sum()
message = "({})".format(cost)
message += "".join(code2char[code] for code in sample)
if sample == target:
message += " CORRECT!"
messages.append((cost, message))
messages.sort(key=lambda tuple_: -tuple_[0])
for _, message in messages:
print(message)
def main():
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
parser = argparse.ArgumentParser(
"Case study of generating simple 1d sequences with RNN.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"mode", choices=["train", "plot"],
help="The mode to run. Use `train` to train a new model"
" and `plot` to plot a sequence generated by an"
" existing one.")
parser.add_argument(
"prefix", default="sine",
help="The prefix for model, timing and state files")
parser.add_argument(
"--input-noise", type=float, default=0.0,
help="Adds Gaussian noise of given intensity to the "
" training sequences.")
parser.add_argument(
"--function", default="lambda a, x: numpy.sin(a * x)",
help="An analytical description of the sequence family to learn."
" The arguments before the last one are considered parameters.")
parser.add_argument(
"--steps", type=int, default=100,
help="Number of steps to plot")
parser.add_argument(
"--params",
help="Parameter values for plotting")
args = parser.parse_args()
function = eval(args.function)
num_params = len(inspect.getargspec(function).args) - 1
class Emitter(TrivialEmitter):
@application
def cost(self, readouts, outputs):
"""Compute MSE."""
return ((readouts - outputs) ** 2).sum(axis=readouts.ndim - 1)
transition = GatedRecurrent(
name="transition", activation=Tanh(), dim=10,
weights_init=Orthogonal())
with_params = AddParameters(transition, num_params, "params",
name="with_params")
generator = SequenceGenerator(
LinearReadout(readout_dim=1, source_names=["states"],
emitter=Emitter(name="emitter"), name="readout"),
with_params,
weights_init=IsotropicGaussian(0.01), biases_init=Constant(0),
name="generator")
generator.allocate()
logger.debug("Parameters:\n" +
pprint.pformat(
[(key, value.get_value().shape) for key, value
in Selector(generator).get_params().items()],
width=120))
if args.mode == "train":
seed = 1
rng = numpy.random.RandomState(seed)
batch_size = 10
generator.initialize()
cost = ComputationGraph(
generator.cost(tensor.tensor3('x'),
params=tensor.matrix("params")).sum())
cost = apply_noise(cost, cost.inputs, args.input_noise)
gh_model = GroundhogModel(generator, cost)
state = GroundhogState(args.prefix, batch_size,
learning_rate=0.0001).as_dict()
data = SeriesIterator(rng, function, 100, batch_size)
trainer = SGD(gh_model, state, data)
main_loop = MainLoop(data, None, None, gh_model, trainer, state, None)
main_loop.load()
main_loop.main()
elif args.mode == "plot":
load_params(generator, args.prefix + "model.npz")
params = tensor.matrix("params")
sample = theano.function([params], generator.generate(
params=params, n_steps=args.steps, batch_size=1))
param_values = numpy.array(map(float, args.params.split()),
dtype=floatX)
states, outputs, _ = sample(param_values[None, :])
actual = outputs[:, 0, 0]
desired = numpy.array([function(*(list(param_values) + [T]))
for T in range(args.steps)])
print("MSE: {}".format(((actual - desired) ** 2).sum()))
pyplot.plot(numpy.hstack([actual[:, None], desired[:, None]]))
pyplot.show()
else:
assert False
source_names=["states"],
emitter=SoftmaxEmitter(name="emitter"),
feedback_brick=LookupFeedback(alphabet_size,
feedback_dim=alphabet_size,
name="feedback"),
name="readout")
seq_gen = SequenceGenerator(readout=readout,
transition=rnn,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
name="generator")
seq_gen.push_initialization_config()
rnn.weights_init = Orthogonal()
seq_gen.initialize()
# z markov_tutorial
x = tensor.lvector('features')
x = x.reshape((x.shape[0], 1))
cost = aggregation.mean(seq_gen.cost_matrix(x[:, :]).sum(), x.shape[1])
cost.name = "negative log-likelihood"
cost_cg = ComputationGraph(cost)
print VariableFilter(roles=[WEIGHT])(cost_cg.variables)
# theano.printing.pydotprint(cost, outfile="./pics/symbolic_graph_unopt.png", var_with_name_simple=True)
algorithm = GradientDescent(cost=cost,
parameters=list(
Selector(seq_gen).get_parameters().values()),
step_rule=Scale(0.001))
def main(mode, save_path, steps, num_batches, load_params):
chars = (list(string.ascii_uppercase) + list(range(10)) +
[' ', '.', ',', '\'', '"', '!', '?', '<UNK>'])
char_to_ind = {char: i for i, char in enumerate(chars)}
ind_to_char = {v: k for k, v in char_to_ind.iteritems()}
train_dataset = TextFile(['/Tmp/serdyuk/data/wsj_text_train'],
char_to_ind, bos_token=None, eos_token=None,
level='character')
valid_dataset = TextFile(['/Tmp/serdyuk/data/wsj_text_valid'],
char_to_ind, bos_token=None, eos_token=None,
level='character')
vocab_size = len(char_to_ind)
logger.info('Dictionary size: {}'.format(vocab_size))
if mode == 'continue':
continue_training(save_path)
return
elif mode == "sample":
main_loop = load(open(save_path, "rb"))
generator = main_loop.model.get_top_bricks()[-1]
sample = ComputationGraph(generator.generate(
n_steps=steps, batch_size=1, iterate=True)).get_theano_function()
states, outputs, costs = [data[:, 0] for data in sample()]
print("".join([ind_to_char[s] for s in outputs]))
numpy.set_printoptions(precision=3, suppress=True)
print("Generation cost:\n{}".format(costs.sum()))
freqs = numpy.bincount(outputs).astype(floatX)
freqs /= freqs.sum()
trans_freqs = numpy.zeros((vocab_size, vocab_size), dtype=floatX)
for a, b in zip(outputs, outputs[1:]):
trans_freqs[a, b] += 1
trans_freqs /= trans_freqs.sum(axis=1)[:, None]
return
# Experiment configuration
batch_size = 20
dim = 650
feedback_dim = 650
valid_stream = valid_dataset.get_example_stream()
valid_stream = Batch(valid_stream,
iteration_scheme=ConstantScheme(batch_size))
valid_stream = Padding(valid_stream)
valid_stream = Mapping(valid_stream, _transpose)
# Build the bricks and initialize them
transition = GatedRecurrent(name="transition", dim=dim,
activation=Tanh())
generator = SequenceGenerator(
Readout(readout_dim=vocab_size, source_names=transition.apply.states,
emitter=SoftmaxEmitter(name="emitter"),
feedback_brick=LookupFeedback(
vocab_size, feedback_dim, name='feedback'),
name="readout"),
transition,
weights_init=Uniform(std=0.04), biases_init=Constant(0),
name="generator")
generator.push_initialization_config()
transition.weights_init = Orthogonal()
transition.push_initialization_config()
generator.initialize()
# Build the cost computation graph.
features = tensor.lmatrix('features')
features_mask = tensor.matrix('features_mask')
cost_matrix = generator.cost_matrix(
features, mask=features_mask)
batch_cost = cost_matrix.sum()
cost = aggregation.mean(
batch_cost,
features.shape[1])
cost.name = "sequence_log_likelihood"
char_cost = aggregation.mean(
batch_cost, features_mask.sum())
char_cost.name = 'character_log_likelihood'
ppl = 2 ** (cost / numpy.log(2))
ppl.name = 'ppl'
bits_per_char = char_cost / tensor.log(2)
bits_per_char.name = 'bits_per_char'
length = features.shape[0]
length.name = 'length'
model = Model(batch_cost)
if load_params:
params = load_parameter_values(save_path)
model.set_parameter_values(params)
if mode == "train":
# Give an idea of what's going on.
logger.info("Parameters:\n" +
pprint.pformat(
[(key, value.get_value().shape) for key, value
in Selector(generator).get_parameters().items()],
width=120))
train_stream = train_dataset.get_example_stream()
train_stream = Mapping(train_stream, _truncate)
train_stream = Batch(train_stream,
iteration_scheme=ConstantScheme(batch_size))
train_stream = Padding(train_stream)
train_stream = Mapping(train_stream, _transpose)
parameters = model.get_parameter_dict()
maxnorm_subjects = VariableFilter(roles=[WEIGHT])(parameters.values())
algorithm = GradientDescent(
cost=batch_cost,
parameters=parameters.values(),
step_rule=CompositeRule([StepClipping(1000.),
AdaDelta(epsilon=1e-8) #, Restrict(VariableClipping(1.0, axis=0), maxnorm_subjects)
]))
ft = features[:6, 0]
ft.name = 'feature_example'
observables = [cost, ppl, char_cost, length, bits_per_char]
for name, param in parameters.items():
num_elements = numpy.product(param.get_value().shape)
norm = param.norm(2) / num_elements ** 0.5
grad_norm = algorithm.gradients[param].norm(2) / num_elements ** 0.5
step_norm = algorithm.steps[param].norm(2) / num_elements ** 0.5
stats = tensor.stack(norm, grad_norm, step_norm, step_norm / grad_norm)
stats.name = name + '_stats'
observables.append(stats)
track_the_best_bpc = TrackTheBest('valid_bits_per_char')
root_path, extension = os.path.splitext(save_path)
this_step_monitoring = TrainingDataMonitoring(
observables + [ft], prefix="this_step", after_batch=True)
average_monitoring = TrainingDataMonitoring(
observables + [algorithm.total_step_norm,
algorithm.total_gradient_norm],
prefix="average",
every_n_batches=10)
valid_monitoring = DataStreamMonitoring(
observables, prefix="valid",
every_n_batches=1500, before_training=False,
data_stream=valid_stream)
main_loop = MainLoop(
algorithm=algorithm,
data_stream=train_stream,
model=model,
extensions=[
this_step_monitoring,
average_monitoring,
valid_monitoring,
track_the_best_bpc,
Checkpoint(save_path, ),
Checkpoint(save_path,
every_n_batches=500,
save_separately=["model", "log"],
use_cpickle=True)
.add_condition(
['after_epoch'],
OnLogRecord(track_the_best_bpc.notification_name),
(root_path + "_best" + extension,)),
Timing(after_batch=True),
Printing(every_n_batches=10),
Plot(root_path,
[[average_monitoring.record_name(cost),
valid_monitoring.record_name(cost)],
[average_monitoring.record_name(algorithm.total_step_norm)],
[average_monitoring.record_name(algorithm.total_gradient_norm)],
[average_monitoring.record_name(ppl),
valid_monitoring.record_name(ppl)],
[average_monitoring.record_name(char_cost),
valid_monitoring.record_name(char_cost)],
[average_monitoring.record_name(bits_per_char),
valid_monitoring.record_name(bits_per_char)]],
every_n_batches=10)
])
main_loop.run()
elif mode == 'evaluate':
with open('/data/lisatmp3/serdyuk/wsj_lms/lms/wsj_trigram_with_initial_eos/lexicon.txt') as f:
raw_words = [line.split()[1:-1] for line in f.readlines()]
words = [[char_to_ind[c] if c in char_to_ind else char_to_ind['<UNK>'] for c in w]
for w in raw_words]
max_word_length = max([len(w) for w in words])
initial_states = tensor.matrix('init_states')
cost_matrix_step = generator.cost_matrix(features, mask=features_mask,
states=initial_states)
cg = ComputationGraph(cost_matrix_step)
states = cg.auxiliary_variables[-2]
compute_cost = theano.function([features, features_mask, initial_states],
[cost_matrix_step.sum(axis=0), states])
cost_matrix = generator.cost_matrix(features, mask=features_mask)
initial_cg = ComputationGraph(cost_matrix)
initial_states = initial_cg.auxiliary_variables[-2]
total_word_cost = 0
num_words = 0
examples = numpy.zeros((max_word_length + 1, len(words)),
dtype='int64')
all_masks = numpy.zeros((max_word_length + 1, len(words)),
dtype=floatX)
for i, word in enumerate(words):
examples[:len(word), i] = word
all_masks[:len(word), i] = 1.
single_space = numpy.array([char_to_ind[' ']])[:, None]
for batch in valid_stream.get_epoch_iterator():
for example, mask in equizip(batch[0].T, batch[1].T):
example = example[:(mask.sum())]
spc_inds = list(numpy.where(example == char_to_ind[" "])[0])
state = generator.transition.transition.initial_states_.get_value()[None, :]
for i, j in equizip([-1] + spc_inds, spc_inds + [-1]):
word = example[(i+1):j, None]
word_cost, states = compute_cost(
word, numpy.ones_like(word, dtype=floatX), state)
state = states[-1]
costs = numpy.exp(-compute_cost(
examples, all_masks, numpy.tile(state, [examples.shape[1], 1]))[0])
_, space_states = compute_cost(
single_space, numpy.ones_like(single_space, dtype=floatX), state)
state = space_states[-1]
word_prob = numpy.exp(-word_cost)
total_word_cost += word_cost + numpy.log(numpy.sum(costs))
num_words += 1
print(word_prob)
print(numpy.sum(costs))
print("Average cost", total_word_cost / num_words)
print("PPL", numpy.exp(total_word_cost / num_words))
print("Word-level perplexity")
print(total_word_cost / num_words)
else:
assert False
def main(mode, save_path, steps, num_batches):
num_states = MarkovChainDataset.num_states
if mode == "train":
# Experiment configuration
rng = numpy.random.RandomState(1)
batch_size = 50
seq_len = 100
dim = 10
feedback_dim = 8
# Build the bricks and initialize them
transition = GatedRecurrent(name="transition", dim=dim,
activation=Tanh())
generator = SequenceGenerator(
Readout(readout_dim=num_states, source_names=["states"],
emitter=SoftmaxEmitter(name="emitter"),
feedback_brick=LookupFeedback(
num_states, feedback_dim, name='feedback'),
name="readout"),
transition,
weights_init=IsotropicGaussian(0.01), biases_init=Constant(0),
name="generator")
generator.push_initialization_config()
transition.weights_init = Orthogonal()
generator.initialize()
# Give an idea of what's going on.
logger.info("Parameters:\n" +
pprint.pformat(
[(key, value.get_value().shape) for key, value
in Selector(generator).get_params().items()],
width=120))
logger.info("Markov chain entropy: {}".format(
MarkovChainDataset.entropy))
logger.info("Expected min error: {}".format(
-MarkovChainDataset.entropy * seq_len))
# Build the cost computation graph.
x = tensor.lmatrix('data')
cost = aggregation.mean(generator.cost_matrix(x[:, :]).sum(),
x.shape[1])
cost.name = "sequence_log_likelihood"
algorithm = GradientDescent(
cost=cost, params=list(Selector(generator).get_params().values()),
step_rule=Scale(0.001))
main_loop = MainLoop(
algorithm=algorithm,
data_stream=DataStream(
MarkovChainDataset(rng, seq_len),
iteration_scheme=ConstantScheme(batch_size)),
model=Model(cost),
extensions=[FinishAfter(after_n_batches=num_batches),
TrainingDataMonitoring([cost], prefix="this_step",
after_batch=True),
TrainingDataMonitoring([cost], prefix="average",
every_n_batches=100),
Checkpoint(save_path, every_n_batches=500),
Printing(every_n_batches=100)])
main_loop.run()
elif mode == "sample":
main_loop = cPickle.load(open(save_path, "rb"))
generator = main_loop.model
sample = ComputationGraph(generator.generate(
n_steps=steps, batch_size=1, iterate=True)).get_theano_function()
states, outputs, costs = [data[:, 0] for data in sample()]
numpy.set_printoptions(precision=3, suppress=True)
print("Generation cost:\n{}".format(costs.sum()))
freqs = numpy.bincount(outputs).astype(theano.config.floatX)
freqs /= freqs.sum()
print("Frequencies:\n {} vs {}".format(freqs,
MarkovChainDataset.equilibrium))
trans_freqs = numpy.zeros((num_states, num_states),
dtype=theano.config.floatX)
for a, b in zip(outputs, outputs[1:]):
trans_freqs[a, b] += 1
trans_freqs /= trans_freqs.sum(axis=1)[:, None]
print("Transition frequencies:\n{}\nvs\n{}".format(
trans_freqs, MarkovChainDataset.trans_prob))
else:
assert False
source_names=source_names + ["feedback"] + ["glimpses"],
emitter=emitter,
feedback_brick=feedback,
name="readout",
)
generator = SequenceGenerator(readout=readout, transition=transition, attention=attention, name="generator")
generator.weights_init = IsotropicGaussian(0.01)
generator.biases_init = Constant(0.0)
generator.push_initialization_config()
generator.transition.biases_init = IsotropicGaussian(0.01, 1)
generator.transition.push_initialization_config()
generator.initialize()
lookup.weights_init = IsotropicGaussian(0.001)
lookup.biases_init = Constant(0.0)
lookup.initialize()
# states = {}
states = [state for state in generator.transition.apply.outputs if state != "step"]
# ipdb.set_trace()
states = {name: shared_floatx_zeros((batch_size, hidden_size_recurrent)) for name in states}
cost_matrix = generator.cost_matrix(x, attended=context, **states)
cost = cost_matrix.mean() + 0.0 * start_flag
def main():
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
parser = argparse.ArgumentParser(
"Case study of language modeling with RNN",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"mode", choices=["train", "sample"],
help="The mode to run. Use `train` to train a new model"
" and `sample` to sample a sequence generated by an"
" existing one.")
parser.add_argument(
"prefix", default="sine",
help="The prefix for model, timing and state files")
parser.add_argument(
"state", nargs="?", default="",
help="Changes to Groundhog state")
parser.add_argument("--path", help="Path to a language dataset")
parser.add_argument("--dict", help="Path to the dataset dictionary")
parser.add_argument("--restart", help="Start anew")
parser.add_argument(
"--reset", action="store_true", default=False,
help="Reset the hidden state between batches")
parser.add_argument(
"--steps", type=int, default=100,
help="Number of steps to plot for the 'sample' mode"
" OR training sequence length for the 'train' mode.")
args = parser.parse_args()
logger.debug("Args:\n" + str(args))
dim = 200
num_chars = 50
transition = GatedRecurrent(
name="transition", activation=Tanh(), dim=dim,
weights_init=Orthogonal())
generator = SequenceGenerator(
LinearReadout(readout_dim=num_chars, source_names=["states"],
emitter=SoftmaxEmitter(name="emitter"),
feedbacker=LookupFeedback(
num_chars, dim, name='feedback'),
name="readout"),
transition,
weights_init=IsotropicGaussian(0.01), biases_init=Constant(0),
name="generator")
generator.allocate()
logger.debug("Parameters:\n" +
pprint.pformat(
[(key, value.get_value().shape) for key, value
in Selector(generator).get_params().items()],
width=120))
if args.mode == "train":
batch_size = 1
seq_len = args.steps
generator.initialize()
# Build cost computation graph that uses the saved hidden states.
# An issue: for Groundhog this is completely transparent, that's
# why it does not carry the hidden state over the period when
# validation in done. We should find a way to fix in the future.
x = tensor.lmatrix('x')
init_states = shared_floatx_zeros((batch_size, dim),
name='init_states')
reset = tensor.scalar('reset')
cost = ComputationGraph(
generator.cost(x, states=init_states * reset).sum())
# TODO: better search routine
states = [v for v in cost.variables
if hasattr(v.tag, 'application_call')
and v.tag.application_call.brick == generator.transition
and (v.tag.application_call.application ==
generator.transition.apply)
and v.tag.role == VariableRole.OUTPUT
and v.tag.name == 'states']
assert len(states) == 1
states = states[0]
gh_model = GroundhogModel(generator, cost)
gh_model.properties.append(
('bpc', cost.outputs[0] * numpy.log(2) / seq_len))
gh_model.properties.append(('mean_init_state', init_states.mean()))
gh_model.properties.append(('reset', reset))
if not args.reset:
gh_model.updates.append((init_states, states[-1]))
state = GroundhogState(args.prefix, batch_size,
learning_rate=0.0001).as_dict()
changes = eval("dict({})".format(args.state))
state.update(changes)
def output_format(x, y, reset):
return dict(x=x[:, None], reset=reset)
train, valid, test = [
LMIterator(batch_size=batch_size,
use_infinite_loop=mode == 'train',
path=args.path,
seq_len=seq_len,
mode=mode,
chunks='chars',
output_format=output_format,
can_fit=True)
for mode in ['train', 'valid', 'test']]
trainer = SGD(gh_model, state, train)
state['on_nan'] = 'warn'
state['cutoff'] = 1.
main_loop = MainLoop(train, valid, None, gh_model,
trainer, state, None)
if not args.restart:
main_loop.load()
main_loop.main()
elif args.mode == "sample":
load_params(generator, args.prefix + "model.npz")
chars = numpy.load(args.dict)['unique_chars']
sample = ComputationGraph(generator.generate(
n_steps=args.steps, batch_size=10, iterate=True)).function()
states, outputs, costs = sample()
for i in range(10):
print("Generation cost: {}".format(costs[:, i].sum()))
print("".join([chars[o] for o in outputs[:, i]]))
else:
assert False
def main(mode, save_path, steps, time_budget, reset):
num_states = ChainDataset.num_states
if mode == "train":
# Experiment configuration
rng = numpy.random.RandomState(1)
batch_size = 50
seq_len = 100
dim = 10
feedback_dim = 8
# Build the bricks and initialize them
transition = GatedRecurrent(name="transition", activation=Tanh(),
dim=dim)
generator = SequenceGenerator(
LinearReadout(readout_dim=num_states, source_names=["states"],
emitter=SoftmaxEmitter(name="emitter"),
feedbacker=LookupFeedback(
num_states, feedback_dim, name='feedback'),
name="readout"),
transition,
weights_init=IsotropicGaussian(0.01), biases_init=Constant(0),
name="generator")
generator.push_initialization_config()
transition.weights_init = Orthogonal()
generator.initialize()
logger.info("Parameters:\n" +
pprint.pformat(
[(key, value.get_value().shape) for key, value
in Selector(generator).get_params().items()],
width=120))
logger.info("Markov chain entropy: {}".format(
ChainDataset.entropy))
logger.info("Expected min error: {}".format(
-ChainDataset.entropy * seq_len * batch_size))
if os.path.isfile(save_path) and not reset:
model = Pylearn2Model.load(save_path)
else:
model = Pylearn2Model(generator)
# Build the cost computation graph.
# Note: would be probably nicer to make cost part of the model.
x = tensor.ltensor3('x')
cost = Pylearn2Cost(model.brick.cost(x[:, :, 0]).sum())
dataset = ChainDataset(rng, seq_len)
sgd = SGD(learning_rate=0.0001, cost=cost,
batch_size=batch_size, batches_per_iter=10,
monitoring_dataset=dataset,
monitoring_batch_size=batch_size,
monitoring_batches=1,
learning_rule=Pylearn2LearningRule(
SGDLearningRule(),
dict(training_objective=cost.cost)))
train = Pylearn2Train(dataset, model, algorithm=sgd,
save_path=save_path, save_freq=10)
train.main_loop(time_budget=time_budget)
elif mode == "sample":
model = Pylearn2Model.load(save_path)
generator = model.brick
sample = ComputationGraph(generator.generate(
n_steps=steps, batch_size=1, iterate=True)).function()
states, outputs, costs = [data[:, 0] for data in sample()]
numpy.set_printoptions(precision=3, suppress=True)
print("Generation cost:\n{}".format(costs.sum()))
freqs = numpy.bincount(outputs).astype(floatX)
freqs /= freqs.sum()
print("Frequencies:\n {} vs {}".format(freqs,
ChainDataset.equilibrium))
trans_freqs = numpy.zeros((num_states, num_states), dtype=floatX)
for a, b in zip(outputs, outputs[1:]):
trans_freqs[a, b] += 1
trans_freqs /= trans_freqs.sum(axis=1)[:, None]
print("Transition frequencies:\n{}\nvs\n{}".format(
trans_freqs, ChainDataset.trans_prob))
else:
assert False
def test_sequence_generator():
"""Test a sequence generator with no contexts and continuous outputs.
Such sequence generators can be used to model e.g. dynamical systems.
"""
rng = numpy.random.RandomState(1234)
output_dim = 1
dim = 20
batch_size = 30
n_steps = 10
transition = SimpleRecurrent(activation=Tanh(), dim=dim,
weights_init=Orthogonal())
generator = SequenceGenerator(
Readout(readout_dim=output_dim, source_names=["states"],
emitter=TestEmitter()),
transition,
weights_init=IsotropicGaussian(0.1), biases_init=Constant(0.0),
seed=1234)
generator.initialize()
# Test 'cost_matrix' method
y = tensor.tensor3('y')
mask = tensor.matrix('mask')
costs = generator.cost_matrix(y, mask)
assert costs.ndim == 2
y_test = rng.uniform(size=(n_steps, batch_size, output_dim)).astype(floatX)
m_test = numpy.ones((n_steps, batch_size), dtype=floatX)
costs_val = theano.function([y, mask], [costs])(y_test, m_test)[0]
assert costs_val.shape == (n_steps, batch_size)
assert_allclose(costs_val.sum(), 115.593, rtol=1e-5)
# Test 'cost' method
cost = generator.cost(y, mask)
assert cost.ndim == 0
cost_val = theano.function([y, mask], [cost])(y_test, m_test)
assert_allclose(cost_val, 3.8531, rtol=1e-5)
# Test 'AUXILIARY' variable 'per_sequence_element' in 'cost' method
cg = ComputationGraph([cost])
var_filter = VariableFilter(roles=[AUXILIARY])
aux_var_name = '_'.join([generator.name, generator.cost.name,
'per_sequence_element'])
cost_per_el = [el for el in var_filter(cg.variables)
if el.name == aux_var_name][0]
assert cost_per_el.ndim == 0
cost_per_el_val = theano.function([y, mask], [cost_per_el])(y_test, m_test)
assert_allclose(cost_per_el_val, 0.38531, rtol=1e-5)
# Test 'generate' method
states, outputs, costs = [variable.eval() for variable in
generator.generate(
states=rng.uniform(
size=(batch_size, dim)).astype(floatX),
iterate=True, batch_size=batch_size,
n_steps=n_steps)]
assert states.shape == (n_steps, batch_size, dim)
assert outputs.shape == (n_steps, batch_size, output_dim)
assert costs.shape == (n_steps, batch_size)
assert_allclose(outputs.sum(), -0.33683, rtol=1e-5)
assert_allclose(states.sum(), 15.7909, rtol=1e-5)
# There is no generation cost in this case, since generation is
# deterministic
assert_allclose(costs.sum(), 0.0)
def __init__(self, config, vocab_size):
context = tensor.imatrix('context')
context_mask = tensor.imatrix('context_mask')
answer = tensor.imatrix('answer')
answer_mask = tensor.imatrix('answer_mask')
bricks = []
context = context.dimshuffle(1, 0)
context_mask = context_mask.dimshuffle(1, 0)
answer = answer.dimshuffle(1, 0)
answer_mask = answer_mask.dimshuffle(1, 0)
context_bag = to_bag(context, vocab_size)
# Embed questions and context
embed = LookupTable(vocab_size, config.embed_size, name='embed')
embed.weights_init = IsotropicGaussian(0.01)
#embeddings_initial_value = init_embedding_table(filename='embeddings/vocab_embeddings.txt')
#embed.weights_init = Constant(embeddings_initial_value)
# Calculate context encoding (concatenate layer1)
cembed = embed.apply(context)
clstms, chidden_list = make_bidir_lstm_stack(
cembed, config.embed_size,
context_mask.astype(theano.config.floatX), config.ctx_lstm_size,
config.ctx_skip_connections, 'ctx')
bricks = bricks + clstms
if config.ctx_skip_connections:
cenc_dim = 2 * sum(config.ctx_lstm_size) #2 : fw & bw
cenc = tensor.concatenate(chidden_list, axis=2)
else:
cenc_dim = 2 * config.ctx_lstm_size[-1]
cenc = tensor.concatenate(chidden_list[-2:], axis=2)
cenc.name = 'cenc'
# Build the encoder bricks
transition = GatedRecurrent(activation=Tanh(),
dim=config.generator_lstm_size,
name="transition")
attention = SequenceContentAttention(
state_names=transition.apply.states,
attended_dim=cenc_dim,
match_dim=config.generator_lstm_size,
name="attention")
readout = Readout(readout_dim=vocab_size,
source_names=[
transition.apply.states[0],
attention.take_glimpses.outputs[0]
],
emitter=MaskedSoftmaxEmitter(context_bag=context_bag,
name='emitter'),
feedback_brick=LookupFeedback(
vocab_size, config.feedback_size),
name="readout")
generator = SequenceGenerator(readout=readout,
transition=transition,
attention=attention,
name="generator")
cost = generator.cost(answer,
answer_mask.astype(theano.config.floatX),
attended=cenc,
attended_mask=context_mask.astype(
theano.config.floatX),
name="cost")
self.predictions = generator.generate(
n_steps=7,
batch_size=config.batch_size,
attended=cenc,
attended_mask=context_mask.astype(theano.config.floatX),
iterate=True)[1]
# Apply dropout
cg = ComputationGraph([cost])
if config.w_noise > 0:
noise_vars = VariableFilter(roles=[WEIGHT])(cg)
cg = apply_noise(cg, noise_vars, config.w_noise)
if config.dropout > 0:
cg = apply_dropout(cg, chidden_list, config.dropout)
[cost_reg] = cg.outputs
# Other stuff
cost.name = 'cost'
cost_reg.name = 'cost_reg'
self.sgd_cost = cost_reg
self.monitor_vars = [[cost_reg]]
self.monitor_vars_valid = [[cost_reg]]
# initialize new stuff manually (change!)
generator.weights_init = IsotropicGaussian(0.01)
generator.biases_init = Constant(0)
generator.push_allocation_config()
generator.push_initialization_config()
transition.weights_init = Orthogonal()
generator.initialize()
# Initialize bricks
embed.initialize()
for brick in bricks:
brick.weights_init = config.weights_init
brick.biases_init = config.biases_init
brick.initialize()
def main(mode, save_path, steps, num_batches):
num_states = MarkovChainDataset.num_states
if mode == "train":
# Experiment configuration
rng = numpy.random.RandomState(1)
batch_size = 50
seq_len = 100
dim = 10
feedback_dim = 8
# Build the bricks and initialize them
transition = GatedRecurrent(name="transition",
dim=dim,
activation=Tanh())
generator = SequenceGenerator(Readout(
readout_dim=num_states,
source_names=["states"],
emitter=SoftmaxEmitter(name="emitter"),
feedback_brick=LookupFeedback(num_states,
feedback_dim,
name='feedback'),
name="readout"),
transition,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
name="generator")
generator.push_initialization_config()
transition.weights_init = Orthogonal()
generator.initialize()
# Give an idea of what's going on.
logger.info("Parameters:\n" + pprint.pformat(
[(key, value.get_value().shape)
for key, value in Selector(generator).get_params().items()],
width=120))
logger.info("Markov chain entropy: {}".format(
MarkovChainDataset.entropy))
logger.info("Expected min error: {}".format(
-MarkovChainDataset.entropy * seq_len))
# Build the cost computation graph.
x = tensor.lmatrix('data')
cost = aggregation.mean(
generator.cost_matrix(x[:, :]).sum(), x.shape[1])
cost.name = "sequence_log_likelihood"
algorithm = GradientDescent(
cost=cost,
params=list(Selector(generator).get_params().values()),
step_rule=Scale(0.001))
main_loop = MainLoop(algorithm=algorithm,
data_stream=DataStream(
MarkovChainDataset(rng, seq_len),
iteration_scheme=ConstantScheme(batch_size)),
model=Model(cost),
extensions=[
FinishAfter(after_n_batches=num_batches),
TrainingDataMonitoring([cost],
prefix="this_step",
after_batch=True),
TrainingDataMonitoring([cost],
prefix="average",
every_n_batches=100),
Checkpoint(save_path, every_n_batches=500),
Printing(every_n_batches=100)
])
main_loop.run()
elif mode == "sample":
main_loop = cPickle.load(open(save_path, "rb"))
generator = main_loop.model
sample = ComputationGraph(
generator.generate(n_steps=steps, batch_size=1,
iterate=True)).get_theano_function()
states, outputs, costs = [data[:, 0] for data in sample()]
numpy.set_printoptions(precision=3, suppress=True)
print("Generation cost:\n{}".format(costs.sum()))
freqs = numpy.bincount(outputs).astype(floatX)
freqs /= freqs.sum()
print("Frequencies:\n {} vs {}".format(freqs,
MarkovChainDataset.equilibrium))
trans_freqs = numpy.zeros((num_states, num_states), dtype=floatX)
for a, b in zip(outputs, outputs[1:]):
trans_freqs[a, b] += 1
trans_freqs /= trans_freqs.sum(axis=1)[:, None]
print("Transition frequencies:\n{}\nvs\n{}".format(
trans_freqs, MarkovChainDataset.trans_prob))
else:
assert False
def test_sequence_generator_with_lm():
floatX = theano.config.floatX
rng = numpy.random.RandomState(1234)
readout_dim = 5
feedback_dim = 3
dim = 20
batch_size = 30
n_steps = 10
transition = GatedRecurrent(dim=dim,
activation=Tanh(),
weights_init=Orthogonal())
language_model = SequenceGenerator(Readout(
readout_dim=readout_dim,
source_names=["states"],
emitter=SoftmaxEmitter(theano_seed=1234),
feedback_brick=LookupFeedback(readout_dim, dim, name='feedback')),
SimpleRecurrent(dim, Tanh()),
name='language_model')
generator = SequenceGenerator(Readout(
readout_dim=readout_dim,
source_names=["states", "lm_states"],
emitter=SoftmaxEmitter(theano_seed=1234),
feedback_brick=LookupFeedback(readout_dim, feedback_dim)),
transition,
language_model=language_model,
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0),
seed=1234)
generator.initialize()
# Test 'cost_matrix' method
y = tensor.lmatrix('y')
y.tag.test_value = numpy.zeros((15, batch_size), dtype='int64')
mask = tensor.matrix('mask')
mask.tag.test_value = numpy.ones((15, batch_size))
costs = generator.cost_matrix(y, mask)
assert costs.ndim == 2
costs_fun = theano.function([y, mask], [costs])
y_test = rng.randint(readout_dim, size=(n_steps, batch_size))
m_test = numpy.ones((n_steps, batch_size), dtype=floatX)
costs_val = costs_fun(y_test, m_test)[0]
assert costs_val.shape == (n_steps, batch_size)
assert_allclose(costs_val.sum(), 483.153, rtol=1e-5)
# Test 'cost' method
cost = generator.cost(y, mask)
assert cost.ndim == 0
cost_val = theano.function([y, mask], cost)(y_test, m_test)
assert_allclose(cost_val, 16.105, rtol=1e-5)
# Test 'AUXILIARY' variable 'per_sequence_element' in 'cost' method
cg = ComputationGraph([cost])
var_filter = VariableFilter(roles=[AUXILIARY])
aux_var_name = '_'.join(
[generator.name, generator.cost.name, 'per_sequence_element'])
cost_per_el = [
el for el in var_filter(cg.variables) if el.name == aux_var_name
][0]
assert cost_per_el.ndim == 0
cost_per_el_val = theano.function([y, mask], [cost_per_el])(y_test, m_test)
assert_allclose(cost_per_el_val, 1.61051, rtol=1e-5)
# Test generate
states, outputs, lm_states, costs = generator.generate(
iterate=True, batch_size=batch_size, n_steps=n_steps)
cg = ComputationGraph([states, outputs, costs])
states_val, outputs_val, costs_val = theano.function(
[], [states, outputs, costs], updates=cg.updates)()
assert states_val.shape == (n_steps, batch_size, dim)
assert outputs_val.shape == (n_steps, batch_size)
assert outputs_val.dtype == 'int64'
assert costs_val.shape == (n_steps, batch_size)
assert_allclose(states_val.sum(), -4.88367, rtol=1e-5)
assert_allclose(costs_val.sum(), 486.681, rtol=1e-5)
assert outputs_val.sum() == 627
# Test masks agnostic results of cost
cost1 = costs_fun([[1], [2]], [[1], [1]])[0]
cost2 = costs_fun([[3, 1], [4, 2], [2, 0]], [[1, 1], [1, 1], [1, 0]])[0]
assert_allclose(cost1.sum(), cost2[:, 1].sum(), rtol=1e-5)
emitter=emitter,
feedback_brick=feedback,
name="readout")
attention = SimpleSequenceAttention(state_names=source_names,
state_dims=[hidden_size_recurrent],
attended_dim=context_size)
generator = SequenceGenerator(readout=readout,
transition=transition,
attention=attention,
name="generator")
generator.weights_init = IsotropicGaussian(0.01)
generator.biases_init = Constant(0.)
generator.initialize()
mlp_context.weights_init = IsotropicGaussian(0.01)
mlp_context.biases_init = Constant(0.)
mlp_context.initialize()
#ipdb.set_trace()
cost_matrix = generator.cost_matrix(x,
x_mask,
attended=mlp_context.apply(context))
cost = cost_matrix.sum() / x_mask.sum()
cost.name = "sequence_log_likelihood"
cg = ComputationGraph(cost)
model = Model(cost)
def main():
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
parser = argparse.ArgumentParser(
"Case study of generating simple 1d sequences with RNN.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"mode",
choices=["train", "plot"],
help="The mode to run. Use `train` to train a new model"
" and `plot` to plot a sequence generated by an"
" existing one.")
parser.add_argument("prefix",
default="sine",
help="The prefix for model, timing and state files")
parser.add_argument("--input-noise",
type=float,
default=0.0,
help="Adds Gaussian noise of given intensity to the "
" training sequences.")
parser.add_argument(
"--function",
default="lambda a, x: numpy.sin(a * x)",
help="An analytical description of the sequence family to learn."
" The arguments before the last one are considered parameters.")
parser.add_argument("--steps",
type=int,
default=100,
help="Number of steps to plot")
parser.add_argument("--params", help="Parameter values for plotting")
args = parser.parse_args()
function = eval(args.function)
num_params = len(inspect.getargspec(function).args) - 1
class Emitter(TrivialEmitter):
@application
def cost(self, readouts, outputs):
"""Compute MSE."""
return ((readouts - outputs)**2).sum(axis=readouts.ndim - 1)
transition = GatedRecurrent(name="transition",
activation=Tanh(),
dim=10,
weights_init=Orthogonal())
with_params = AddParameters(transition,
num_params,
"params",
name="with_params")
generator = SequenceGenerator(LinearReadout(
readout_dim=1,
source_names=["states"],
emitter=Emitter(name="emitter"),
name="readout"),
with_params,
weights_init=IsotropicGaussian(0.01),
biases_init=Constant(0),
name="generator")
generator.allocate()
logger.debug("Parameters:\n" + pprint.pformat(
[(key, value.get_value().shape)
for key, value in Selector(generator).get_params().items()],
width=120))
if args.mode == "train":
seed = 1
rng = numpy.random.RandomState(seed)
batch_size = 10
generator.initialize()
cost = ComputationGraph(
generator.cost(tensor.tensor3('x'),
params=tensor.matrix("params")).sum())
cost = apply_noise(cost, cost.inputs, args.input_noise)
gh_model = GroundhogModel(generator, cost)
state = GroundhogState(args.prefix, batch_size,
learning_rate=0.0001).as_dict()
data = SeriesIterator(rng, function, 100, batch_size)
trainer = SGD(gh_model, state, data)
main_loop = MainLoop(data, None, None, gh_model, trainer, state, None)
main_loop.load()
main_loop.main()
elif args.mode == "plot":
load_params(generator, args.prefix + "model.npz")
params = tensor.matrix("params")
sample = theano.function([params],
generator.generate(params=params,
n_steps=args.steps,
batch_size=1))
param_values = numpy.array(map(float, args.params.split()),
dtype=floatX)
states, outputs, _ = sample(param_values[None, :])
actual = outputs[:, 0, 0]
desired = numpy.array(
[function(*(list(param_values) + [T])) for T in range(args.steps)])
print("MSE: {}".format(((actual - desired)**2).sum()))
pyplot.plot(numpy.hstack([actual[:, None], desired[:, None]]))
pyplot.show()
else:
assert False
