def test_assign_dev_data():
config = Config()
config.update(dummyconfig_dict)
device = DummyDevice(config=config)
dataset = DummyDataset(input_dim=config.int("num_inputs", 0),
output_dim=config.int("num_outputs", 0),
num_seqs=10)
batches = [generate_batch(0, dataset), generate_batch(1, dataset)]
success, num_batches = assign_dev_data(device, dataset, batches)
assert_true(success)
assert_equal(num_batches, len(batches))
def test_DummyDevice():
dataset = Task12AXDataset(num_seqs=1000,
seq_ordering="random",
chunking="200:200")
dataset.init_seq_order(epoch=1)
batch_gen = dataset.generate_batches(recurrent_net=True,
batch_size=1000,
max_seqs=3)
batches = batch_gen.peek_next_n(1)
dev = DummyDevice()
assign_success, _ = engine_util.assign_dev_data(device=dev,
dataset=dataset,
batches=batches)
assert assign_success
def forward_fill_queue(self):
"""
Full sequence forwarding, no chunking (at the moment).
"""
assert self.train_started
if self.is_forwarding_finished: return
# We will ignore max_seq_length.
batch_size = self.config.int('batch_size', 1)
max_seqs = self.config.int('max_seqs', -1)
if max_seqs <= 0: max_seqs = float('inf')
dataset = self.engine.train_data
from returnn.engine.batch import Batch
# Collect all batches.
forward_batches = []; ":type: list[EngineBatch.Batch]"
num_seqs = 0
while self._device_exec("have_space_in_forward_data_queue", num_seqs=num_seqs):
# Load next sequence for forwarding, keep all which are still needed for training.
if not dataset.is_less_than_num_seqs(self.forward_current_seq):
self.is_forwarding_finished = True
break
dataset.load_seqs(self.train_start_seq, self.forward_current_seq + 1)
seq_len = dataset.get_seq_length(self.forward_current_seq)
if not forward_batches:
forward_batches.append(Batch())
batch = forward_batches[-1]
dt, ds = batch.try_sequence_as_slice(seq_len)
if ds > 1 and ((dt * ds).max_value() > batch_size or ds > max_seqs):
batch = Batch()
forward_batches.append(batch)
batch.add_sequence_as_slice(seq_idx=self.forward_current_seq, seq_start_frame=0, length=seq_len)
num_seqs += 1
self.forward_current_seq += 1
# Forward the batches.
from returnn.theano.engine_util import assign_dev_data
for batch in forward_batches:
print("SeqTrainParallelControl, forward %r" % batch, file=log.v4)
success = assign_dev_data(self.train_device, dataset, [batch], load_seqs=False)
assert success, "failed to allocate & assign data"
self.train_device.update_data()
self._device_exec("do_forward", batch=batch)
self._device_exec("train_check_calc_loss")
def test_assign_dev_data():
config = Config()
config.update(dummyconfig_dict)
print("Create ExternSprintDataset")
dataset = ExternSprintDataset([
sys.executable, sprintExecPath
], "--*.feature-dimension=2 --*.trainer-output-dimension=3 --*.crnn-dataset=DummyDataset(2,3,num_seqs=4,seq_len=10)"
)
dataset.init_seq_order(epoch=1)
assert_true(dataset.is_less_than_num_seqs(0))
recurrent = False
batch_generator = dataset.generate_batches(recurrent_net=recurrent,
batch_size=5)
batches = batch_generator.peek_next_n(2)
assert_equal(len(batches), 2)
if theano:
print("Create Device")
device = DummyDevice(config=config)
success, num_batches = assign_dev_data(device, dataset, batches)
assert_true(success)
assert_equal(num_batches, len(batches))
def load(lstm_opts=None):
if not lstm_opts: lstm_opts = {"class": "lstm2"}
lstm_opts = lstm_opts.copy()
lstm_opts.update({"n_out": 10, "from": "in"})
num_inputs = 9
num_outputs = 2
net_topo = {
"in": {
"class": "dump",
"filename": "in"
},
"lstm": lstm_opts,
"lstm_dump": {
"class": "dump",
"from": "lstm",
"filename": "lstm"
},
"output": {
"class": "softmax",
"loss": "ce",
"from": "lstm_dump"
}
}
collected_data = {}
DumpLayer.global_debug_container = collected_data
net = network.LayerNetwork.from_json(json_content=net_topo,
n_in=num_inputs,
n_out={"classes": (num_outputs, 1)},
train_flag=True)
net.declare_train_params()
# Init dataset and prepare one minibatch.
epoch = 1
dataset = Task12AXDataset(num_seqs=1000,
seq_ordering="random",
chunking="200:200")
dataset.init_seq_order(epoch=epoch)
batch_gen = dataset.generate_batches(recurrent_net=net.recurrent,
batch_size=5000,
max_seqs=10)
batches = batch_gen.peek_next_n(1)
# We need the DummyDevice for assign_dev_data.
dev = DummyDevice()
assign_success, _ = engine_util.assign_dev_data(device=dev,
dataset=dataset,
batches=batches)
assert assign_success
dev.initialize(net)
dev.update_data()
givens = [(net.y[k], dev.y[k]) for k in dev.used_data_keys]
givens += [(net.j[k], dev.j[k]) for k in dev.used_data_keys]
# Now gradients, updates and compile everything.
gradients = {
p: T.grad(net.get_objective(), p, known_grads=net.known_grads)
for p in net.train_params_vars
}
updater = Updater(adam=True)
updater.initVars(net, gradients)
updater.setLearningRate(learning_rate=0.01)
trainer = theano.function(inputs=[],
outputs=[net.total_cost],
givens=givens,
updates=updater.getUpdateList(),
on_unused_input='warn',
name="train_and_updater")
for p in net.train_params_vars:
collected_data["param:%s" % p.name] = p.get_value()
# And finally, run it.
cost = trainer()
collected_data["cost"] = cost
return collected_data
def assign_dev_data(self, device, batches):
return assign_dev_data(device, self.data, batches)