def __init__(self, config=None, blocking=True):
if not config:
config = Config()
config.update(dummyconfig_dict)
super(DummyDevice, self).__init__(device="cpu",
config=config,
blocking=blocking)
def test_stftConfig_multi_res_02():
with make_scope() as session:
layer_name = "stft_layer"
fft_sizes = [400, 200, 800]
frame_sizes = [400, 200, 800]
frame_shift = 160
window = "hanning"
test_input = np.random.normal(0, 0.6, (1, 3200, 2))
num_outputs = int(
np.sum([(int(fft_size / 2) + 1) * test_input.shape[2]
for fft_size in fft_sizes]))
config = Config()
config.update({
"num_outputs": num_outputs,
"num_inputs": test_input.shape[2],
"network": {
layer_name: {
"class": "multichannel_multiresolution_stft_layer",
"frame_shift": frame_shift,
"frame_sizes": frame_sizes,
"window": window,
"fft_sizes": fft_sizes,
"use_rfft": True,
"nr_of_channels": 2,
"is_output_layer": True,
"from": "data:data"
}
}
})
network = TFNetwork(config=config, train_flag=True)
network.construct_from_dict(config.typed_value("network"))
layer = network.layers[layer_name]
test_output = session.run(
layer.output.placeholder,
{network.get_extern_data('data').placeholder: test_input})
assert test_output.shape[2] == num_outputs
comparison_frame = 6
ref00 = _get_ref_output_single_res(test_input, fft_sizes[0],
frame_sizes[0], frame_shift,
window, comparison_frame, 0)
ref01 = _get_ref_output_single_res(test_input, fft_sizes[0],
frame_sizes[0], frame_shift,
window, comparison_frame, 1)
ref10 = _get_ref_output_single_res(test_input, fft_sizes[1],
frame_sizes[1], frame_shift,
window, comparison_frame, 0)
ref11 = _get_ref_output_single_res(test_input, fft_sizes[1],
frame_sizes[1], frame_shift,
window, comparison_frame, 1)
ref20 = _get_ref_output_single_res(test_input, fft_sizes[2],
frame_sizes[2], frame_shift,
window, comparison_frame, 0)
ref21 = _get_ref_output_single_res(test_input, fft_sizes[2],
frame_sizes[2], frame_shift,
window, comparison_frame, 1)
ref = np.concatenate([ref00, ref01, ref10, ref11, ref20, ref21],
axis=0)
resultDiff = np.abs(test_output[0, comparison_frame, :] - ref)
assert np.mean(resultDiff) < 0.06
assert np.max(resultDiff) < 1
def test_NetworkDescription_to_json_config1():
config = Config()
config.update(config1_dict)
desc = LayerNetworkDescription.from_config(config)
desc_json_content = desc.to_json_content()
pprint(desc_json_content)
assert_in("hidden_0", desc_json_content)
assert_equal(desc_json_content["hidden_0"]["class"], "forward")
assert_in("hidden_1", desc_json_content)
assert_in("output", desc_json_content)
orig_network = LayerNetwork.from_description(desc)
assert_in("hidden_0", orig_network.hidden)
assert_in("hidden_1", orig_network.hidden)
assert_equal(len(orig_network.hidden), 2)
assert_is_instance(orig_network.hidden["hidden_0"], ForwardLayer)
assert_equal(orig_network.hidden["hidden_0"].layer_class, "hidden")
orig_json_content = orig_network.to_json_content()
pprint(orig_json_content)
assert_in("hidden_0", orig_json_content)
assert_equal(orig_json_content["hidden_0"]["class"], "hidden")
assert_in("hidden_1", orig_json_content)
assert_in("output", orig_json_content)
new_network = LayerNetwork.from_json(
desc_json_content, config1_dict["num_inputs"],
{"classes": (config1_dict["num_outputs"], 1)})
new_json_content = new_network.to_json_content()
if orig_json_content != new_json_content:
print(dict_diff_str(orig_json_content, new_json_content))
assert_equal(orig_json_content, new_network.to_json_content())
def test_melFilterbankLayer():
with make_scope() as session:
n_in, n_out = 257, 3
layer_name = "mel_filterbank_layer"
config = Config()
config.update({
"num_outputs": n_out,
"num_inputs": n_in,
"network": {
layer_name: {
"class": "mel_filterbank",
"fft_size": 512,
"nr_of_filters": n_out,
"n_out": n_out,
"is_output_layer": True,
"from": "data:data"
}
}
})
network = TFNetwork(config=config, train_flag=True)
network.construct_from_dict(config.typed_value("network"))
layer = network.layers[layer_name]
test_out = session.run(layer.output.placeholder,
feed_dict={
network.get_extern_data('data').placeholder:
np.ones((1, 1, 257))
})
assert np.sum(test_out - np.asarray(
[28.27923584, 53.10634232, 99.71585846], dtype=np.float32)) < 1e-5
def test_config1_basic():
config = Config()
config.update(config1_dict)
desc = LayerNetworkDescription.from_config(config)
assert_is_instance(desc.hidden_info, list)
assert_equal(len(desc.hidden_info), len(config1_dict["hidden_size"]))
assert_equal(desc.num_inputs, config1_dict["num_inputs"])
def test_assign_dev_data():
config = Config()
config.update(dummyconfig_dict)
device = DummyDevice(config=config)
dataset = DummyDataset(input_dim=config.int("num_inputs", 0),
output_dim=config.int("num_outputs", 0),
num_seqs=10)
batches = [generate_batch(0, dataset), generate_batch(1, dataset)]
success, num_batches = assign_dev_data(device, dataset, batches)
assert_true(success)
assert_equal(num_batches, len(batches))
def test_num_inputs_outputs_old():
n_in = 5
n_out = 10
config = Config()
config.update({"num_inputs": n_in, "num_outputs": n_out})
num_inputs, num_outputs = LayerNetworkDescription.num_inputs_outputs_from_config(
config)
assert_equal(num_inputs, n_in)
assert_is_instance(num_outputs, dict)
assert_equal(len(num_outputs), 1)
assert_in("classes", num_outputs)
assert_equal(num_outputs["classes"], (n_out, 1))
def test_config1():
config = Config()
config.update(config1_dict)
pretrain = pretrain_from_config(config)
assert_equal(pretrain.get_train_num_epochs(), 2)
net1_json = pretrain.get_network_json_for_epoch(1)
net2_json = pretrain.get_network_json_for_epoch(2)
net3_json = pretrain.get_network_json_for_epoch(3)
assert_in("hidden_0", net1_json)
assert_not_in("hidden_1", net1_json)
assert_in("hidden_0", net2_json)
assert_in("hidden_1", net2_json)
assert_equal(net2_json, net3_json)
def test_init_error_muliple_out():
config = Config()
config.update({
"learning_rate_control": "newbob",
"learning_rate_control_error_measure": "dev_score"
})
lrc = load_learning_rate_control_from_config(config)
assert isinstance(lrc, NewbobRelative)
lrc.get_learning_rate_for_epoch(1)
lrc.set_epoch_error(
1, {"train_score": {
'cost:output': 1.95,
"cost:out2": 2.95
}})
lrc.set_epoch_error(
1, {
"dev_score": {
'cost:output': 1.99,
"cost:out2": 2.99
},
"dev_error": {
'error:output': 0.6,
"error:out2": 0.7
}
})
error = lrc.get_epoch_error_dict(1)
assert "train_score_output" in error
assert "train_score_out2" in error
assert "dev_score_output" in error
assert "dev_score_out2" in error
assert "dev_error_output" in error
assert "dev_error_out2" in error
assert_equal(lrc.get_error_key(1), "dev_score_output")
lrc.get_learning_rate_for_epoch(2)
lrc.set_epoch_error(
2, {"train_score": {
'cost:output': 1.8,
"cost:out2": 2.8
}})
lrc.set_epoch_error(
2, {
"dev_score": {
'cost:output': 1.9,
"cost:out2": 2.9
},
"dev_error": {
'error:output': 0.5,
"error:out2": 0.6
}
})
lrc.get_learning_rate_for_epoch(3)
def test_config1_to_json_network_copy():
config = Config()
config.update(config1_dict)
orig_network = LayerNetwork.from_config_topology(config)
orig_json_content = orig_network.to_json_content()
pprint(orig_json_content)
new_network = LayerNetwork.from_json(orig_json_content, orig_network.n_in,
orig_network.n_out)
assert_equal(orig_network.n_in, new_network.n_in)
assert_equal(orig_network.n_out, new_network.n_out)
new_json_content = new_network.to_json_content()
if orig_json_content != new_json_content:
print(dict_diff_str(orig_json_content, new_json_content))
assert_equal(orig_json_content, new_network.to_json_content())
def test_config2_bidirect_lstm():
config = Config()
config.update(config2_dict)
desc = LayerNetworkDescription.from_config(config)
assert_true(desc.bidirectional)
network = LayerNetwork.from_config_topology(config)
net_json = network.to_json_content()
pprint(net_json)
assert_in("output", net_json)
assert_in("hidden_0_fw", net_json)
assert_in("hidden_0_bw", net_json)
assert_in("hidden_1_fw", net_json)
assert_in("hidden_1_bw", net_json)
assert_in("hidden_2_fw", net_json)
assert_in("hidden_2_bw", net_json)
assert_equal(net_json["output"]["from"], ["hidden_2_fw", "hidden_2_bw"])
assert_equal(len(net_json), 7)
def test_Device_blocking_init():
config = Config()
config.update({
"multiprocessing": False,
"blocking": True,
"device": "cpu",
"num_epochs": 1,
"num_inputs": 3,
"num_outputs": 2,
})
config.network_topology_json = """
{
"output": {"class": "softmax", "loss": "ce"}
}
"""
Device("cpu", config=config, blocking=True)
def test_network_config1_init():
config = Config()
config.update(config1_dict)
network = LayerNetwork.from_config_topology(config)
assert_in("hidden_0", network.hidden)
assert_in("hidden_1", network.hidden)
assert_equal(len(network.hidden), 2)
assert_is_instance(network.hidden["hidden_0"], ForwardLayer)
assert_equal(network.hidden["hidden_0"].layer_class, "hidden")
assert_false(network.recurrent)
json_content = network.to_json_content()
pprint(json_content)
assert_in("hidden_0", json_content)
assert_equal(json_content["hidden_0"]["class"], "hidden")
assert_in("hidden_1", json_content)
assert_in("output", json_content)
def test_rfftStftConfig_01():
with make_scope() as session:
layer_name = "stft_layer"
fft_size = 400
frame_size = 400
frame_shift = 160
window = "hanning"
test_input = np.ones((1, 32000, 2), dtype=np.float32)
config = Config()
config.update({
"num_outputs":
int(fft_size / 2) + 1 * test_input.shape[2],
"num_inputs":
test_input.shape[2],
"network": {
layer_name: {
"class": "multichannel_stft_layer",
"frame_shift": frame_shift,
"frame_size": frame_size,
"window": window,
"fft_size": fft_size,
"use_rfft": True,
"nr_of_channels": 2,
"is_output_layer": True,
"from": "data:data"
}
}
})
network = TFNetwork(config=config, train_flag=True)
network.construct_from_dict(config.typed_value("network"))
layer = network.layers[layer_name]
test_output = session.run(
layer.output.placeholder,
{network.get_extern_data('data').placeholder: test_input})
ref0 = _get_ref_output(test_input, fft_size, frame_size,
frame_shift, window, 0, 0)
# np.fft.rfft and tensorflow.python.ops.rfft differ a little bit in their
# results, thus an error margin is allowed in the result
resultDiff = np.abs(test_output[0, 0, 0:(int(fft_size / 2) + 1)] -
ref0)
assert np.mean(resultDiff) < 0.02
assert np.max(resultDiff) < 1
pass
def test_single_default_target_init():
config_single_default = Config()
config_single_default.update({
"multiprocessing": False,
"blocking": True,
"device": "cpu",
"num_epochs": 1,
"num_inputs": 3,
"num_outputs": 2,
})
config_single_default.network_topology_json = """
{
"output": {"class": "softmax", "loss": "ce"}
}
"""
dev = Device("cpu", config=config_single_default, blocking=True)
num_params = get_num_params(dev.trainnet.get_all_params_vars())
assert_equal(num_params, 3 * 2 + 2, "W, b")
def test_init_error_old():
config = Config()
config.update({
"learning_rate_control": "newbob",
"learning_rate_control_error_measure": "dev_score"
})
lrc = load_learning_rate_control_from_config(config)
assert isinstance(lrc, NewbobRelative)
lrc.get_learning_rate_for_epoch(1)
lrc.set_epoch_error(1, {"train_score": 1.9344199658230012})
lrc.set_epoch_error(1, {"dev_score": 1.99, "dev_error": 0.6})
error = lrc.get_epoch_error_dict(1)
assert "train_score" in error
assert "dev_score" in error
assert "dev_error" in error
assert_equal(lrc.get_error_key(1), "dev_score")
lrc.get_learning_rate_for_epoch(2)
lrc.set_epoch_error(2, {"train_score": 1.8})
lrc.set_epoch_error(2, {"dev_score": 1.9, "dev_error": 0.5})
lrc.get_learning_rate_for_epoch(3)
def test_stftConfig_single_res_01():
with make_scope() as session:
layer_name = "stft_layer"
fft_sizes = [400]
frame_sizes = [400]
frame_shift = 160
window = "hanning"
test_input = np.ones((1, 32000, 2), dtype=np.float32)
num_outputs = (int(fft_sizes[0] / 2) + 1) * test_input.shape[2]
config = Config()
config.update({
"num_outputs": num_outputs,
"num_inputs": test_input.shape[2],
"network": {
layer_name: {
"class": "multichannel_multiresolution_stft_layer",
"frame_shift": frame_shift,
"frame_sizes": frame_sizes,
"window": window,
"fft_sizes": fft_sizes,
"use_rfft": True,
"nr_of_channels": 2,
"is_output_layer": True,
"from": "data:data"
}
}
})
network = TFNetwork(config=config, train_flag=True)
network.construct_from_dict(config.typed_value("network"))
layer = network.layers[layer_name]
test_output = session.run(
layer.output.placeholder,
{network.get_extern_data('data').placeholder: test_input})
ref0 = _get_ref_output_single_res(test_input, fft_sizes[0],
frame_sizes[0], frame_shift,
window, 0, 0)
resultDiff = np.abs(test_output[0, 0, 0:(int(fft_sizes[0] / 2) +
1)] - ref0)
assert test_output.shape[2] == num_outputs
assert np.mean(resultDiff) < 0.02
assert np.max(resultDiff) < 1
def test_newbob_multi_epoch():
lr = 0.0005
config = Config()
config.update({
"learning_rate_control": "newbob_multi_epoch",
"learning_rate_control_relative_error_relative_lr": True,
"newbob_multi_num_epochs": 6,
"newbob_multi_update_interval": 1,
"learning_rate": lr
})
lrc = load_learning_rate_control_from_config(config)
assert isinstance(lrc, NewbobMultiEpoch)
assert_equal(lrc.get_learning_rate_for_epoch(1), lr)
lrc.set_epoch_error(
1, {
'dev_error': 0.50283176046904721,
'dev_score': 2.3209858321263455,
'train_score': 3.095824052426714,
})
assert_equal(lrc.get_learning_rate_for_epoch(2),
lr) # epoch 2 cannot be a different lr yet
def test_assign_dev_data():
config = Config()
config.update(dummyconfig_dict)
print("Create ExternSprintDataset")
dataset = ExternSprintDataset([
sys.executable, sprintExecPath
], "--*.feature-dimension=2 --*.trainer-output-dimension=3 --*.crnn-dataset=DummyDataset(2,3,num_seqs=4,seq_len=10)"
)
dataset.init_seq_order(epoch=1)
assert_true(dataset.is_less_than_num_seqs(0))
recurrent = False
batch_generator = dataset.generate_batches(recurrent_net=recurrent,
batch_size=5)
batches = batch_generator.peek_next_n(2)
assert_equal(len(batches), 2)
if theano:
print("Create Device")
device = DummyDevice(config=config)
success, num_batches = assign_dev_data(device, dataset, batches)
assert_true(success)
assert_equal(num_batches, len(batches))
def test_read_all():
config = Config()
config.update(dummyconfig_dict)
print("Create ExternSprintDataset")
python2_exec = util.which("python2")
if python2_exec is None:
raise unittest.SkipTest("python2 not found")
num_seqs = 4
dataset = ExternSprintDataset(
[python2_exec, sprintExecPath],
"--*.feature-dimension=2 --*.trainer-output-dimension=3 "
"--*.crnn-dataset=DummyDataset(2,3,num_seqs=%i,seq_len=10)" % num_seqs)
dataset.init_seq_order(epoch=1)
seq_idx = 0
while dataset.is_less_than_num_seqs(seq_idx):
dataset.load_seqs(seq_idx, seq_idx + 1)
for key in dataset.get_data_keys():
value = dataset.get_data(seq_idx, key)
print("seq idx %i, data %r: %r" % (seq_idx, key, value))
seq_idx += 1
assert seq_idx == num_seqs
def test_later_default_lr():
import tempfile
tmp_file = tempfile.mktemp()
lr = 0.0005
learning_rates = list(numpy.linspace(0.0003, lr, num=10)) # warmup
config = Config()
config.update({
"learning_rate_file": tmp_file,
"learning_rate_control": "newbob_multi_epoch",
"learning_rate_control_relative_error_relative_lr": True,
"learning_rate_control_min_num_epochs_per_new_lr": 3,
"newbob_multi_num_epochs": 6,
"newbob_multi_update_interval": 1,
"learning_rate": lr,
"learning_rates": learning_rates,
"min_learning_rate": lr / 50.
})
lrc = load_learning_rate_control_from_config(config)
assert isinstance(lrc, NewbobMultiEpoch)
num_epochs = 250
for epoch in range(1, num_epochs + 1):
lrc.get_learning_rate_for_epoch(epoch)
lrc.set_epoch_error(epoch, {"train_score": 0.5, "train_error": 0.5})
lrc.set_epoch_error(epoch, {"dev_score": 0.5, "dev_error": 0.5})
print("Learning rates:")
print(lrc)
lrc.save()
print("Saved to:", lrc.filename)
learning_rates = {i + 1: v for (i, v) in enumerate(learning_rates)}
later_epoch = num_epochs + 1
learning_rates[later_epoch] = lr * 0.5 # reset
config.update({"learning_rates": learning_rates})
lrc = load_learning_rate_control_from_config(config)
assert later_epoch in lrc.epoch_data
lr251 = lrc.get_learning_rate_for_epoch(later_epoch)
numpy.testing.assert_almost_equal(lr251, learning_rates[later_epoch])
def test_num_inputs_outputs_special_dataset():
config = Config()
config.update({
"train": {
"class": "CopyTaskDataset",
"num_seqs": 1000,
"nsymbols": 80,
"minlen": 100,
"maxlen": 100
},
"num_outputs": {
"data": [80, 1],
"classes": [80, 1]
}
})
num_inputs, num_outputs = LayerNetworkDescription.num_inputs_outputs_from_config(
config)
assert_equal(num_inputs, 80)
assert_in("data", num_outputs)
assert_in("classes", num_outputs)
assert_equal(num_outputs["classes"], (80, 1))
assert_equal(num_outputs["data"], (80, 1))
def test_complexLinearProjectionLayer():
with make_scope() as session:
n_in, n_out = 514, 128
layer_name = "clp_layer"
config = Config()
config.update({
"num_outputs": n_out,
"num_inputs": n_in,
"network": {
layer_name: {
"class": "complex_linear_projection",
"nr_of_filters": n_out,
"n_out": n_out,
"is_output_layer": True,
"from": "data:data"
}
}
})
network = TFNetwork(config=config, train_flag=True)
network.construct_from_dict(config.typed_value("network"))
layer = network.layers[layer_name]
assert isinstance(layer, ComplexLinearProjectionLayer)
i_r = np.ones((1, n_in // 2))
i_i = np.ones((1, n_in // 2)) * 0.5
test_input = np.expand_dims(
np.reshape(
np.transpose(
np.reshape(np.concatenate([i_r, i_i], axis=1),
(1, 2, 257)), [0, 2, 1]), (1, 514)), 0)
test_clp_kernel = np.ones((2, n_in // 2, 128))
network.initialize_params(session)
test_clp_output = session.run(
layer.output.placeholder,
feed_dict={
network.get_extern_data('data').placeholder: test_input,
layer._clp_kernel: test_clp_kernel
})
assert test_clp_output[0, 0, 0] - 6.00722122 < 1e-5
def benchmark(lstm_unit, use_gpu):
"""
:param str lstm_unit: e.g. "LSTMBlock", one of LstmCellTypes
:param bool use_gpu:
:return: runtime in seconds of the training itself, excluding initialization
:rtype: float
"""
device = {True: "GPU", False: "CPU"}[use_gpu]
key = "%s:%s" % (device, lstm_unit)
print(">>> Start benchmark for %s." % key)
config = Config()
config.update(make_config_dict(lstm_unit=lstm_unit, use_gpu=use_gpu))
dataset_kwargs = config.typed_value("train")
Dataset.kwargs_update_from_config(config, dataset_kwargs)
dataset = init_dataset(dataset_kwargs)
engine = Engine(config=config)
engine.init_train_from_config(config=config, train_data=dataset)
print(">>> Start training now for %s." % key)
start_time = time.time()
engine.train()
runtime = time.time() - start_time
print(">>> Runtime of %s: %s" % (key, hms_fraction(runtime)))
engine.finalize()
return runtime
def test_newbob():
lr = 0.01
config = Config()
config.update({"learning_rate_control": "newbob", "learning_rate": lr})
lrc = load_learning_rate_control_from_config(config)
assert isinstance(lrc, NewbobRelative)
assert_equal(lrc.get_learning_rate_for_epoch(1), lr)
lrc.set_epoch_error(1,
{"train_score": {
'cost:output': 1.9344199658230012
}})
lrc.set_epoch_error(1, {
"dev_score": {
'cost:output': 1.99
},
"dev_error": {
'error:output': 0.6
}
})
error = lrc.get_epoch_error_dict(1)
assert "train_score" in error
assert "dev_score" in error
assert "dev_error" in error
assert_equal(lrc.get_error_key(1), "dev_score")
assert_equal(lrc.get_learning_rate_for_epoch(2),
lr) # epoch 2 cannot be a different lr yet
lrc.set_epoch_error(2, {"train_score": {'cost:output': 1.8}})
lrc.set_epoch_error(2, {
"dev_score": {
'cost:output': 1.9
},
"dev_error": {
'error:output': 0.5
}
})
lrc.get_learning_rate_for_epoch(3)
def test_multi_target_init():
config = Config()
config.update({
"multiprocessing": False,
"blocking": True,
"device": "cpu",
"num_epochs": 1,
"num_inputs": 3,
"num_outputs": {
"t1": 4,
"t2": 5
},
"learning_rate": 1.0,
})
config.network_topology_json = """
{
"fw0": {"class": "hidden", "activation": "identity", "n_out": 3},
"out1": {"class": "softmax", "loss": "ce", "target": "t1", "from": ["fw0"]},
"out2": {"class": "softmax", "loss": "ce", "target": "t2", "from": ["fw0"]}
}
"""
device = Device("cpu", config=config, blocking=True)
assert_true(device.trainnet, "train network initialized")
assert_true(device.testnet, "test network initialized")
param_vars = device.trainnet.get_all_params_vars()
print("params:", param_vars)
assert_equal(len(param_vars), 6, "W, b vars for each out, and fw")
num_params = get_num_params(param_vars)
assert_equal(num_params, (3 * 3 + 3) + (3 * 4 + 4) + (3 * 5 + 5),
"W, b for each out, and fw")
assert_in("fw0", device.testnet.hidden)
assert_in("out1", device.testnet.output)
assert_in("out2", device.testnet.output)
assert_is(device.testnet.j["t1"], device.testnet.output["out1"].index)
assert_true(device.updater)
update_list = device.updater.getUpdateList()
print("update list:")
pprint(update_list)
update_dict = dict(update_list)
assert_equal(len(update_dict), len(update_list),
"all params in update list only once")
assert_in("fw0", device.trainnet.hidden)
assert_equal(len(device.trainnet.hidden), 1)
assert_in("W_in_data_fw0", device.trainnet.hidden["fw0"].params)
assert_in("b_fw0", device.trainnet.hidden["fw0"].params)
assert_equal(len(device.trainnet.hidden["fw0"].params), 2)
assert_in("out1", device.trainnet.output)
assert_equal(len(device.trainnet.output), 2)
assert_in("W_in_fw0_out1", device.trainnet.output["out1"].params)
assert_in("b_out1", device.trainnet.output["out1"].params)
assert_equal(len(device.trainnet.output["out1"].params), 2)
assert_in(device.trainnet.hidden["fw0"].params["W_in_data_fw0"],
update_dict)
assert_in(device.trainnet.hidden["fw0"].params["b_fw0"], update_dict)
assert_in(device.trainnet.output["out1"].params["W_in_fw0_out1"],
update_dict)
assert_in(device.trainnet.output["out1"].params["b_out1"], update_dict)
assert_in(device.trainnet.output["out2"].params["W_in_fw0_out2"],
update_dict)
assert_in(device.trainnet.output["out2"].params["b_out2"], update_dict)
# assert_equal(len(update_dict), 6) # updater adds other stuff...
# Set net params.
net_params = {
"fw0": {
"W_in_data_fw0": numpy.identity(3, dtype="float32"),
"b_fw0": numpy.zeros((3, ), dtype="float32")
},
"out1": {
"W_in_fw0_out1":
numpy.arange(0.0, 1.2, 0.1, dtype="float32").reshape((3, 4)),
"b_out1":
numpy.arange(0.0, 4, dtype="float32")
},
"out2": {
"W_in_fw0_out2":
numpy.arange(0.0, 1.5, 0.1, dtype="float32").reshape((3, 5)),
"b_out2":
numpy.arange(0.0, 5, dtype="float32")
}
}
device.trainnet.set_params_by_dict(net_params)
device.testnet.set_params_by_dict(net_params)
# Show params.
for p in param_vars:
print("init %s:" % p)
pprint(p.get_value())
# Init dataset.
dataset = StaticDataset(data=[{
"data":
numpy.array([[0.1, 0.2, -0.3]], dtype="float32"),
"t1":
numpy.array([2]),
"t2":
numpy.array([4])
}],
output_dim=config.typed_value("num_outputs"))
dataset.init_seq_order()
assert_equal(dataset.is_data_sparse("data"), False)
assert_equal(dataset.is_data_sparse("t1"), True)
assert_equal(dataset.is_data_sparse("t2"), True)
# Copy to device allocation.
success = assign_dev_data_single_seq(device, dataset, 0)
assert_true(success, "failed to allocate & assign data")
# Check allocated data.
assert_equal(device.targets["data"].shape,
(1, 1, 3)) # input shape. (time,batch,dim)
assert_in("t1", device.targets)
assert_in("t2", device.targets)
assert_equal(device.targets["t1"].shape, (1, 1))
assert_equal(device.targets["t2"].shape, (1, 1))
assert_equal(device.output_index["data"].shape, (1, 1))
numpy.testing.assert_equal(device.output_index["data"], numpy.array([[1]]))
assert_equal(device.output_index["t1"].shape, (1, 1))
numpy.testing.assert_equal(device.output_index["t1"], numpy.array([[1]]))
# Forward test.
device.update_data()
device.testnet.costs["out1"].name = "out1_cost" # nice in the func graph
out_i1 = device.testnet.output["out1"].index
out_i1_nonzero = device.testnet.output["out1"].i
nll1, pcx1 = T.nnet.crossentropy_softmax_1hot(
x=device.testnet.output["out1"].y_m[out_i1_nonzero],
y_idx=device.testnet.output["out1"].y_data_flat[out_i1_nonzero])
forward_func = theano.function(
inputs=[device.block_start, device.block_end],
outputs=[
device.testnet.j["t1"], out_i1, out_i1_nonzero[0], nll1, pcx1,
device.testnet.costs["out1"],
device.testnet.output["out1"].p_y_given_x,
device.testnet.costs["out2"],
device.testnet.output["out2"].p_y_given_x
],
givens=device.make_givens(device.testnet),
no_default_updates=True,
on_unused_input='warn',
name="forward")
#print "forward func:"
#theano.printing.debugprint(forward_func)
net_j1, out_i1_val, out_i1_nz_val, nll1_val, pcx1_val, t1_cost, t1_y, t2_cost, t2_y = forward_func(
0, 1)
print("forward results:")
pprint(net_j1)
pprint(out_i1_val)
pprint(out_i1_nz_val)
pprint(nll1_val)
pprint(pcx1_val)
pprint(t1_cost)
pprint(t1_y)
pprint(t2_cost)
pprint(t2_y)
assert_equal(net_j1, numpy.array([[1]]))
assert_equal(out_i1_val, numpy.array([[1]]))
assert_equal(out_i1_nz_val, numpy.array([0]))
assert_almost_equal(nll1_val, numpy.array([t1_cost]))
numpy.testing.assert_almost_equal(t1_y, pcx1_val[None, ...])
assert_almost_equal(t1_cost, 1.440189698561195, places=6)
assert_almost_equal(t2_cost, 0.45191439593759336, places=6)
numpy.testing.assert_almost_equal(
t1_y,
numpy.array([[[0.0320586, 0.08714432, 0.23688282, 0.64391426]]]),
decimal=6)
numpy.testing.assert_almost_equal(t2_y,
numpy.array([[[
0.01165623, 0.03168492, 0.08612854,
0.23412166, 0.63640865
]]]),
decimal=6)
# One train step.
device.set_learning_rate(config.typed_value("learning_rate"))
device.run("train")
output_list, outputs_format = device.result()
assert_is_instance(output_list, list)
assert_true(outputs_format, "for train, we should always get the format")
outputs = Device.make_result_dict(output_list, outputs_format)
pprint(outputs)
assert_in("cost:out1", outputs)
assert_greater(outputs["cost:out1"], 0)
assert_almost_equal(outputs["cost:out1"], t1_cost)
# Get net params.
params = device.get_net_train_params(device.trainnet)
references_params = {
"W_in_data_fw0":
numpy.array([[1.00055406e+00, 5.54056978e-04, 5.54056978e-04],
[1.10811396e-03, 1.00110811e+00, 1.10811396e-03],
[-1.66217093e-03, -1.66217093e-03, 9.98337829e-01]]),
"b_fw0":
numpy.array([0.00554057, 0.00554057, 0.00554057]),
"W_in_fw0_out1":
numpy.array([[-0.00320586, 0.09128557, 0.27631172, 0.23560857],
[0.39358828, 0.48257114, 0.75262344, 0.57121715],
[0.80961758, 0.9261433, 0.77106485, 1.29317428]]),
"b_out1":
numpy.array([-0.0320586, 0.91285568, 2.76311718, 2.35608574]),
"W_in_fw0_out2":
numpy.array([[
-1.16562310e-03, 9.68315079e-02, 1.91387146e-01, 2.76587834e-01,
4.36359135e-01
],
[
4.97668754e-01, 5.93663016e-01, 6.82774291e-01,
7.53175669e-01, 9.72718271e-01
],
[
1.00349687e+00, 1.10950548e+00, 1.22583856e+00,
1.37023650e+00, 1.29092259e+00
]]),
"b_out2":
numpy.array(
[-0.01165623, 0.96831508, 1.91387146, 2.76587834, 4.36359135])
}
assert_equal(len(param_vars), len(params))
for p, v in zip(param_vars, params):
print("%s:" % p)
pprint(v)
assert_true(p.name)
numpy.testing.assert_almost_equal(references_params[p.name],
v,
decimal=6)
def test_config2():
config = Config()
config.update(config2_dict)
pretrain = pretrain_from_config(config)
assert_equal(pretrain.get_train_num_epochs(), 3)
def test_config3():
config = Config()
config.update(config3_dict)
config.network_topology_json = config3_json
pretrain = pretrain_from_config(config)
assert_equal(pretrain.get_train_num_epochs(), 3)
def init_config(config_filename=None,
command_line_options=(),
default_config=None,
extra_updates=None):
"""
:param str|None config_filename:
:param list[str]|tuple[str] command_line_options: e.g. ``sys.argv[1:]``
:param dict[str]|None default_config:
:param dict[str]|None extra_updates:
Initializes the global config.
There are multiple sources which are used to init the config:
* ``configFilename``, and maybe first item of ``commandLineOptions`` interpret as config filename
* other options via ``commandLineOptions``
* ``extra_updates``
Note about the order/priority of these:
* ``extra_updates``
* options from ``commandLineOptions``
* ``configFilename``
* config filename from ``commandLineOptions[0]``
* ``extra_updates``
* options from ``commandLineOptions``
``extra_updates`` and ``commandLineOptions`` are used twice so that they are available
when the config is loaded, which thus has access to them, and can e.g. use them via Python code.
However, the purpose is that they overwrite any option from the config;
that is why we apply them again in the end.
``commandLineOptions`` is applied after ``extra_updates`` so that the user has still the possibility
to overwrite anything set by ``extra_updates``.
"""
global config
config = Config()
config_filenames_by_cmd_line = []
if command_line_options:
# Assume that the first argument prefixed with "+" or "-" and all following is not a config file.
i = 0
for arg in command_line_options:
if arg[:1] in "-+":
break
config_filenames_by_cmd_line.append(arg)
i += 1
command_line_options = command_line_options[i:]
if default_config:
config.update(default_config)
if extra_updates:
config.update(extra_updates)
if command_line_options:
config.parse_cmd_args(command_line_options)
if config_filename:
config.load_file(config_filename)
for fn in config_filenames_by_cmd_line:
config.load_file(fn)
if extra_updates:
config.update(extra_updates)
if command_line_options:
config.parse_cmd_args(command_line_options)
# I really don't know where to put this otherwise:
if config.bool("EnableAutoNumpySharedMemPickling", False):
import returnn.util.task_system
returnn.util.task_system.SharedMemNumpyConfig["enabled"] = True
# Server default options
if config.value('task', 'train') == 'server':
config.set('num_inputs', 2)
config.set('num_outputs', 1)
def test_init_config1():
config = Config()
config.update(config1_dict)
pretrain = pretrain_from_config(config)
assert_true(pretrain)
