def forward(segmentName, features):
"""
:param numpy.ndarray features: format (input-feature,time) (via Sprint)
:return numpy.ndarray, format (output-dim,time)
"""
print("Sprint forward", segmentName, features.shape)
start_time = time.time()
assert engine is not None, "not initialized"
assert sprintDataset
# Features are in Sprint format (feature,time).
T = features.shape[1]
assert features.shape == (InputDim, T)
# Fill the data for the current segment.
sprintDataset.shuffle_frames_of_nseqs = 0 # We must not shuffle.
sprintDataset.initSprintEpoch(None) # Reset cache. We don't need old seqs anymore.
sprintDataset.init_seq_order()
seq = sprintDataset.addNewData(features, segmentName=segmentName)
if BackendEngine.is_theano_selected():
# Prepare data for device.
device = engine.devices[0]
success = assign_dev_data_single_seq(device, sprintDataset, seq)
assert success, "failed to allocate & assign data for seq %i, %s" % (seq, segmentName)
# Do the actual forwarding and collect result.
device.run("extract")
result, _ = device.result()
assert result is not None, "Device crashed."
assert len(result) == 1
posteriors = result[0]
elif BackendEngine.is_tensorflow_selected():
posteriors = engine.forward_single(dataset=sprintDataset, seq_idx=seq)
else:
raise NotImplementedError("unknown backend engine")
# If we have a sequence training criterion, posteriors might be in format (time,seq|batch,emission).
if posteriors.ndim == 3:
assert posteriors.shape == (T, 1, OutputDim)
posteriors = posteriors[:, 0]
# Posteriors are in format (time,emission).
assert posteriors.shape == (T, OutputDim)
# Reformat to Sprint expected format (emission,time).
posteriors = posteriors.transpose()
assert posteriors.shape == (OutputDim, T)
stats = (numpy.min(posteriors), numpy.max(posteriors), numpy.mean(posteriors), numpy.std(posteriors))
print("posteriors min/max/mean/std:", stats, "time:", time.time() - start_time)
if numpy.isinf(posteriors).any() or numpy.isnan(posteriors).any():
print("posteriors:", posteriors)
debug_feat_fn = "/tmp/crnn.pid%i.sprintinterface.debug.features.txt" % os.getpid()
debug_post_fn = "/tmp/crnn.pid%i.sprintinterface.debug.posteriors.txt" % os.getpid()
print("Wrote to files %s, %s" % (debug_feat_fn, debug_post_fn))
numpy.savetxt(debug_feat_fn, features)
numpy.savetxt(debug_post_fn, posteriors)
assert False, "Error, posteriors contain invalid numbers."
return posteriors
def _forward(segment_name, features):
"""
:param numpy.ndarray features: format (input-feature,time) (via Sprint)
:return: format (output-dim,time)
:rtype: numpy.ndarray
"""
print("Sprint forward", segment_name, features.shape)
start_time = time.time()
assert engine is not None, "not initialized"
assert sprintDataset
# Features are in Sprint format (feature,time).
num_time = features.shape[1]
assert features.shape == (InputDim, num_time)
dataset, seq_idx = features_to_dataset(features=features, segment_name=segment_name)
if BackendEngine.is_theano_selected():
# Prepare data for device.
device = engine.devices[0]
success = assign_dev_data_single_seq(device, dataset=dataset, seq=seq_idx)
assert success, "failed to allocate & assign data for seq %i, %s" % (seq_idx, segment_name)
# Do the actual forwarding and collect result.
device.run("extract")
result, _ = device.result()
assert result is not None, "Device crashed."
assert len(result) == 1
posteriors = result[0]
elif BackendEngine.is_tensorflow_selected():
posteriors = engine.forward_single(dataset=dataset, seq_idx=seq_idx)
else:
raise NotImplementedError("unknown backend engine")
# If we have a sequence training criterion, posteriors might be in format (time,seq|batch,emission).
if posteriors.ndim == 3:
assert posteriors.shape == (num_time, 1, OutputDim * MaxSegmentLength)
posteriors = posteriors[:, 0]
# Posteriors are in format (time,emission).
assert posteriors.shape == (num_time, OutputDim * MaxSegmentLength)
# Reformat to Sprint expected format (emission,time).
posteriors = posteriors.transpose()
assert posteriors.shape == (OutputDim * MaxSegmentLength, num_time)
stats = (numpy.min(posteriors), numpy.max(posteriors), numpy.mean(posteriors), numpy.std(posteriors))
print("posteriors min/max/mean/std:", stats, "time:", time.time() - start_time)
if numpy.isinf(posteriors).any() or numpy.isnan(posteriors).any():
print("posteriors:", posteriors)
debug_feat_fn = "/tmp/crnn.pid%i.sprintinterface.debug.features.txt" % os.getpid()
debug_post_fn = "/tmp/crnn.pid%i.sprintinterface.debug.posteriors.txt" % os.getpid()
print("Wrote to files %s, %s" % (debug_feat_fn, debug_post_fn))
numpy.savetxt(debug_feat_fn, features)
numpy.savetxt(debug_post_fn, posteriors)
assert False, "Error, posteriors contain invalid numbers."
return posteriors
def forward(segmentName, features):
"""
:param numpy.ndarray features: format (input-feature,time) (via Sprint)
:return numpy.ndarray, format (output-dim,time)
"""
print "Sprint forward", segmentName, features.shape
assert engine is not None, "not initialized"
assert sprintDataset
# Features are in Sprint format (feature,time).
T = features.shape[1]
assert features.shape == (InputDim, T)
# Fill the data for the current segment.
sprintDataset.shuffle_frames_of_nseqs = 0 # We must not shuffle.
sprintDataset.initSprintEpoch(None) # Reset cache. We don't need old seqs anymore.
sprintDataset.init_seq_order()
seq = sprintDataset.addNewData(features, segmentName=segmentName)
# Prepare data for device.
device = engine.devices[0]
success = assign_dev_data_single_seq(device, sprintDataset, seq)
assert success, "failed to allocate & assign data for seq %i, %s" % (seq, segmentName)
# Do the actual forwarding and collect result.
device.run("extract")
result, _ = device.result()
assert result is not None, "Device crashed."
assert len(result) == 1
posteriors = result[0]
# If we have a sequence training criterion, posteriors might be in format (time,seq|batch,emission).
if posteriors.ndim == 3:
assert posteriors.shape == (T, 1, OutputDim)
posteriors = posteriors[:, 0]
# Posteriors are in format (time,emission).
assert posteriors.shape == (T, OutputDim)
# Reformat to Sprint expected format (emission,time).
posteriors = posteriors.transpose()
assert posteriors.shape == (OutputDim, T)
stats = (numpy.min(posteriors), numpy.max(posteriors), numpy.mean(posteriors), numpy.std(posteriors))
print "posteriors min/max/mean/std:", stats
if numpy.isinf(posteriors).any() or numpy.isnan(posteriors).any():
print "posteriors:", posteriors
debug_feat_fn = "/tmp/crnn.pid%i.sprintinterface.debug.features.txt" % os.getpid()
debug_post_fn = "/tmp/crnn.pid%i.sprintinterface.debug.posteriors.txt" % os.getpid()
print "Wrote to files %s, %s" % (debug_feat_fn, debug_post_fn)
numpy.savetxt(debug_feat_fn, features)
numpy.savetxt(debug_post_fn, posteriors)
assert False, "Error, posteriors contain invalid numbers."
return posteriors
def test_multi_target_init():
config = Config()
config.update({
"multiprocessing": False,
"blocking": True,
"device": "cpu",
"num_epochs": 1,
"num_inputs": 3,
"num_outputs": {
"t1": 4,
"t2": 5
},
"learning_rate": 1.0,
})
config.network_topology_json = """
{
"fw0": {"class": "hidden", "activation": "identity", "n_out": 3},
"out1": {"class": "softmax", "loss": "ce", "target": "t1", "from": ["fw0"]},
"out2": {"class": "softmax", "loss": "ce", "target": "t2", "from": ["fw0"]}
}
"""
device = Device("cpu", config=config, blocking=True)
assert_true(device.trainnet, "train network initialized")
assert_true(device.testnet, "test network initialized")
param_vars = device.trainnet.get_all_params_vars()
print "params:", param_vars
assert_equal(len(param_vars), 6, "W, b vars for each out, and fw")
num_params = get_num_params(param_vars)
assert_equal(num_params, (3 * 3 + 3) + (3 * 4 + 4) + (3 * 5 + 5),
"W, b for each out, and fw")
assert_in("fw0", device.testnet.hidden)
assert_in("out1", device.testnet.output)
assert_in("out2", device.testnet.output)
assert_is(device.testnet.j["t1"], device.testnet.output["out1"].index)
assert_true(device.updater)
update_list = device.updater.getUpdateList()
print "update list:"
pprint(update_list)
update_dict = dict(update_list)
assert_equal(len(update_dict), len(update_list),
"all params in update list only once")
assert_in("fw0", device.trainnet.hidden)
assert_equal(len(device.trainnet.hidden), 1)
assert_in("W_in_data_fw0", device.trainnet.hidden["fw0"].params)
assert_in("b_fw0", device.trainnet.hidden["fw0"].params)
assert_equal(len(device.trainnet.hidden["fw0"].params), 2)
assert_in("out1", device.trainnet.output)
assert_equal(len(device.trainnet.output), 2)
assert_in("W_in_fw0_out1", device.trainnet.output["out1"].params)
assert_in("b_out1", device.trainnet.output["out1"].params)
assert_equal(len(device.trainnet.output["out1"].params), 2)
assert_in(device.trainnet.hidden["fw0"].params["W_in_data_fw0"],
update_dict)
assert_in(device.trainnet.hidden["fw0"].params["b_fw0"], update_dict)
assert_in(device.trainnet.output["out1"].params["W_in_fw0_out1"],
update_dict)
assert_in(device.trainnet.output["out1"].params["b_out1"], update_dict)
assert_in(device.trainnet.output["out2"].params["W_in_fw0_out2"],
update_dict)
assert_in(device.trainnet.output["out2"].params["b_out2"], update_dict)
assert_equal(len(update_dict), 6)
# Set net params.
net_params = {
"fw0": {
"W_in_data_fw0": numpy.identity(3, dtype="float32"),
"b_fw0": numpy.zeros((3, ), dtype="float32")
},
"out1": {
"W_in_fw0_out1":
numpy.arange(0.0, 1.2, 0.1, dtype="float32").reshape((3, 4)),
"b_out1":
numpy.arange(0.0, 4, dtype="float32")
},
"out2": {
"W_in_fw0_out2":
numpy.arange(0.0, 1.5, 0.1, dtype="float32").reshape((3, 5)),
"b_out2":
numpy.arange(0.0, 5, dtype="float32")
}
}
device.trainnet.set_params_by_dict(net_params)
device.testnet.set_params_by_dict(net_params)
# Show params.
for p in param_vars:
print "init %s:" % p
pprint(p.get_value())
# Init dataset.
dataset = StaticDataset(data=[{
"data":
numpy.array([[0.1, 0.2, -0.3]], dtype="float32"),
"t1":
numpy.array([2]),
"t2":
numpy.array([4])
}],
output_dim=config.typed_value("num_outputs"))
dataset.init_seq_order()
assert_equal(dataset.is_data_sparse("data"), False)
assert_equal(dataset.is_data_sparse("t1"), True)
assert_equal(dataset.is_data_sparse("t2"), True)
# Copy to device allocation.
success = assign_dev_data_single_seq(device, dataset, 0)
assert_true(success, "failed to allocate & assign data")
# Check allocated data.
assert_equal(device.targets["data"].shape,
(1, 1, 3)) # input shape. (time,batch,dim)
assert_in("t1", device.targets)
assert_in("t2", device.targets)
assert_equal(device.targets["t1"].shape, (1, 1))
assert_equal(device.targets["t2"].shape, (1, 1))
assert_equal(device.output_index["data"].shape, (1, 1))
numpy.testing.assert_equal(device.output_index["data"], numpy.array([[1]]))
assert_equal(device.output_index["t1"].shape, (1, 1))
numpy.testing.assert_equal(device.output_index["t1"], numpy.array([[1]]))
# Forward test.
device.update_data()
device.testnet.costs["out1"].name = "out1_cost" # nice in the func graph
out_i1 = device.testnet.output["out1"].index
out_i1_nonzero = device.testnet.output["out1"].i
nll1, pcx1 = T.nnet.crossentropy_softmax_1hot(
x=device.testnet.output["out1"].y_m[out_i1_nonzero],
y_idx=device.testnet.output["out1"].y_data_flat[out_i1_nonzero])
forward_func = theano.function(
inputs=[device.block_start, device.block_end],
outputs=[
device.testnet.j["t1"], out_i1, out_i1_nonzero[0], nll1, pcx1,
device.testnet.costs["out1"],
device.testnet.output["out1"].p_y_given_x,
device.testnet.costs["out2"],
device.testnet.output["out2"].p_y_given_x
],
givens=device.make_givens(device.testnet),
no_default_updates=True,
on_unused_input='warn',
name="forward")
#print "forward func:"
#theano.printing.debugprint(forward_func)
net_j1, out_i1_val, out_i1_nz_val, nll1_val, pcx1_val, t1_cost, t1_y, t2_cost, t2_y = forward_func(
0, 1)
print "forward results:"
pprint(net_j1)
pprint(out_i1_val)
pprint(out_i1_nz_val)
pprint(nll1_val)
pprint(pcx1_val)
pprint(t1_cost)
pprint(t1_y)
pprint(t2_cost)
pprint(t2_y)
assert_equal(net_j1, numpy.array([[1]]))
assert_equal(out_i1_val, numpy.array([[1]]))
assert_equal(out_i1_nz_val, numpy.array([0]))
assert_almost_equal(nll1_val, numpy.array([t1_cost]))
numpy.testing.assert_almost_equal(t1_y, pcx1_val)
assert_almost_equal(t1_cost, 1.440189698561195, places=6)
assert_almost_equal(t2_cost, 0.45191439593759336, places=6)
numpy.testing.assert_almost_equal(
t1_y,
numpy.array([[0.0320586, 0.08714432, 0.23688282, 0.64391426]]),
decimal=6)
numpy.testing.assert_almost_equal(t2_y,
numpy.array([[
0.01165623, 0.03168492, 0.08612854,
0.23412166, 0.63640865
]]),
decimal=6)
# One train step.
device.set_learning_rate(config.typed_value("learning_rate"))
device.run("train")
output_list, outputs_format = device.result()
assert_is_instance(output_list, list)
assert_true(outputs_format, "for train, we should always get the format")
outputs = Device.make_result_dict(output_list, outputs_format)
pprint(outputs)
assert_in("cost:out1", outputs)
assert_greater(outputs["cost:out1"], 0)
assert_almost_equal(outputs["cost:out1"], t1_cost)
# Get net params.
params = device.get_net_train_params(device.trainnet)
references_params = {
"W_in_data_fw0":
numpy.array([[1.00055406e+00, 5.54056978e-04, 5.54056978e-04],
[1.10811396e-03, 1.00110811e+00, 1.10811396e-03],
[-1.66217093e-03, -1.66217093e-03, 9.98337829e-01]]),
"b_fw0":
numpy.array([0.00554057, 0.00554057, 0.00554057]),
"W_in_fw0_out1":
numpy.array([[-0.00320586, 0.09128557, 0.27631172, 0.23560857],
[0.39358828, 0.48257114, 0.75262344, 0.57121715],
[0.80961758, 0.9261433, 0.77106485, 1.29317428]]),
"b_out1":
numpy.array([-0.0320586, 0.91285568, 2.76311718, 2.35608574]),
"W_in_fw0_out2":
numpy.array([[
-1.16562310e-03, 9.68315079e-02, 1.91387146e-01, 2.76587834e-01,
4.36359135e-01
],
[
4.97668754e-01, 5.93663016e-01, 6.82774291e-01,
7.53175669e-01, 9.72718271e-01
],
[
1.00349687e+00, 1.10950548e+00, 1.22583856e+00,
1.37023650e+00, 1.29092259e+00
]]),
"b_out2":
numpy.array(
[-0.01165623, 0.96831508, 1.91387146, 2.76587834, 4.36359135])
}
assert_equal(len(param_vars), len(params))
for p, v in zip(param_vars, params):
print "%s:" % p
pprint(v)
assert_true(p.name)
numpy.testing.assert_almost_equal(references_params[p.name],
v,
decimal=6)
def test_combi_auto_enc_longer():
config = Config()
config.update({
"multiprocessing": False,
"blocking": True,
"device": "cpu",
"num_epochs": 1,
"num_inputs": 3,
"num_outputs": {
"classes": 2
},
"learning_rate": 1.0,
"adadelta": True,
"network": {
"output": {
"class": "softmax",
"loss": "ce",
"target": "classes"
},
"auto-enc": {
"class": "softmax",
"loss": "sse",
"dtype": "float32",
"target": "data"
}
}
})
device = Device("cpu", config=config, blocking=True)
# Set net params.
def get_net_params(with_auto_enc=True):
d = {
"output": {
"W_in_data_output":
numpy.arange(0.1, 0.7, 0.1, dtype="float32").reshape((3, 2)),
"b_output":
numpy.arange(0.0, 2, dtype="float32")
}
}
if with_auto_enc:
d["auto-enc"] = {
"W_in_data_auto-enc":
numpy.arange(0.1, 1.0, 0.1, dtype="float32").reshape((3, 3)),
"b_auto-enc":
numpy.arange(0.0, 3, dtype="float32")
}
return d
device.trainnet.set_params_by_dict(get_net_params())
device.testnet.set_params_by_dict(get_net_params())
# Show params.
for p in device.trainnet.get_all_params_vars():
print "init %s:" % p
pprint(p.get_value())
# Init dataset.
dataset = DummyDataset(input_dim=config.typed_value("num_inputs"),
output_dim=config.typed_value("num_outputs"),
num_seqs=10)
dataset.init_seq_order()
cost_output_sum = 0.0
for seq_idx in range(dataset.num_seqs):
# Copy to device allocation.
success = assign_dev_data_single_seq(device, dataset, seq_idx)
assert_true(success, "failed to allocate & assign data")
# One train step.
device.set_learning_rate(config.typed_value("learning_rate"))
device.run("train")
output_list, outputs_format = device.result()
assert_is_instance(output_list, list)
assert_true(outputs_format,
"for train, we should always get the format")
outputs = Device.make_result_dict(output_list, outputs_format)
print("seq %i" % seq_idx)
pprint(outputs)
assert_in("cost:output", outputs)
assert_in("cost:auto-enc", outputs)
cost_output_sum += outputs["cost:output"]
# Now, drop the auto-enc from the network, and redo the same thing.
del config.typed_value("network")["auto-enc"]
device = Device("cpu", config=config, blocking=True)
device.trainnet.set_params_by_dict(get_net_params(with_auto_enc=False))
device.testnet.set_params_by_dict(get_net_params(with_auto_enc=False))
for p in device.trainnet.get_all_params_vars():
print "second run, init %s:" % p
pprint(p.get_value())
dataset.init_seq_order() # reset
cost2_output_sum = 0.0
for seq_idx in range(dataset.num_seqs):
# Copy to device allocation.
success = assign_dev_data_single_seq(device, dataset, seq_idx)
assert_true(success, "failed to allocate & assign data")
# One train step.
device.set_learning_rate(config.typed_value("learning_rate"))
device.run("train")
output_list, outputs_format = device.result()
assert_is_instance(output_list, list)
assert_true(outputs_format,
"for train, we should always get the format")
outputs = Device.make_result_dict(output_list, outputs_format)
print("seq %i" % seq_idx)
pprint(outputs)
assert_in("cost:output", outputs)
assert_not_in("cost:auto-enc", outputs)
cost2_output_sum += outputs["cost:output"]
assert_equal(cost_output_sum, cost2_output_sum)
assert_almost_equal(cost_output_sum, 16.028842568397522, places=6)
def test_combi_auto_enc():
config = Config()
config.update({
"multiprocessing": False,
"blocking": True,
"device": "cpu",
"num_epochs": 1,
"num_inputs": 3,
"num_outputs": {
"classes": 2
},
"learning_rate": 1.0,
"network": {
"output": {
"class": "softmax",
"loss": "ce",
"target": "classes"
},
"auto-enc": {
"class": "softmax",
"loss": "sse",
"dtype": "float32",
"target": "data"
}
}
})
device = Device("cpu", config=config, blocking=True)
# Set net params.
def get_net_params(with_auto_enc=True):
d = {
"output": {
"W_in_data_output":
numpy.arange(0.1, 0.7, 0.1, dtype="float32").reshape((3, 2)),
"b_output":
numpy.arange(0.0, 2, dtype="float32")
}
}
if with_auto_enc:
d["auto-enc"] = {
"W_in_data_auto-enc":
numpy.arange(0.1, 1.0, 0.1, dtype="float32").reshape((3, 3)),
"b_auto-enc":
numpy.arange(0.0, 3, dtype="float32")
}
return d
device.trainnet.set_params_by_dict(get_net_params())
device.testnet.set_params_by_dict(get_net_params())
# Show params.
for p in device.trainnet.get_all_params_vars():
print "init %s:" % p
pprint(p.get_value())
# Init dataset.
dataset = StaticDataset(data=[{
"data":
numpy.array([[0.1, 0.2, -0.3]], dtype="float32"),
"classes":
numpy.array([1]),
}],
output_dim=config.typed_value("num_outputs"))
dataset.init_seq_order()
# Copy to device allocation.
success = assign_dev_data_single_seq(device, dataset, 0)
assert_true(success, "failed to allocate & assign data")
# One train step.
device.set_learning_rate(config.typed_value("learning_rate"))
device.run("train")
output_list, outputs_format = device.result()
assert_is_instance(output_list, list)
assert_true(outputs_format, "for train, we should always get the format")
outputs = Device.make_result_dict(output_list, outputs_format)
pprint(outputs)
assert_in("cost:output", outputs)
assert_in("cost:auto-enc", outputs)
expected_cost_output = 0.3132616877555847
assert_almost_equal(outputs["cost:output"], expected_cost_output, places=6)
exact_cost_output = outputs["cost:output"]
assert_almost_equal(outputs["cost:auto-enc"], 5.263200283050537, places=6)
# Now, drop the auto-enc from the network, and redo the same thing.
del config.typed_value("network")["auto-enc"]
device = Device("cpu", config=config, blocking=True)
device.trainnet.set_params_by_dict(get_net_params(with_auto_enc=False))
device.testnet.set_params_by_dict(get_net_params(with_auto_enc=False))
for p in device.trainnet.get_all_params_vars():
print "second run, init %s:" % p
pprint(p.get_value())
dataset.init_seq_order() # reset. probably not needed
success = assign_dev_data_single_seq(device, dataset, 0)
assert_true(success, "failed to allocate & assign data")
device.set_learning_rate(config.typed_value("learning_rate"))
device.run("train")
output_list, outputs_format = device.result()
assert_is_instance(output_list, list)
assert_true(outputs_format, "for train, we should always get the format")
outputs = Device.make_result_dict(output_list, outputs_format)
pprint(outputs)
assert_in("cost:output", outputs)
assert_not_in("cost:auto-enc", outputs)
assert_almost_equal(outputs["cost:output"], expected_cost_output, places=6)
assert_equal(outputs["cost:output"], exact_cost_output)
def test_multi_target_init():
config = Config()
config.update({
"multiprocessing": False,
"blocking": True,
"device": "cpu",
"num_epochs": 1,
"num_inputs": 3,
"num_outputs": {"t1": 4, "t2": 5},
"learning_rate": 1.0,
})
config.network_topology_json = """
{
"fw0": {"class": "hidden", "activation": "identity", "n_out": 3},
"out1": {"class": "softmax", "loss": "ce", "target": "t1", "from": ["fw0"]},
"out2": {"class": "softmax", "loss": "ce", "target": "t2", "from": ["fw0"]}
}
"""
device = Device("cpu", config=config, blocking=True)
assert_true(device.trainnet, "train network initialized")
assert_true(device.testnet, "test network initialized")
param_vars = device.trainnet.get_all_params_vars()
print "params:", param_vars
assert_equal(len(param_vars), 6, "W, b vars for each out, and fw")
num_params = get_num_params(param_vars)
assert_equal(num_params, (3 * 3 + 3) + (3 * 4 + 4) + (3 * 5 + 5), "W, b for each out, and fw")
assert_in("fw0", device.testnet.hidden)
assert_in("out1", device.testnet.output)
assert_in("out2", device.testnet.output)
assert_is(device.testnet.j["t1"], device.testnet.output["out1"].index)
assert_true(device.updater)
update_list = device.updater.getUpdateList()
print "update list:"
pprint(update_list)
update_dict = dict(update_list)
assert_equal(len(update_dict), len(update_list), "all params in update list only once")
assert_in("fw0", device.trainnet.hidden)
assert_equal(len(device.trainnet.hidden), 1)
assert_in("W_in_data_fw0", device.trainnet.hidden["fw0"].params)
assert_in("b_fw0", device.trainnet.hidden["fw0"].params)
assert_equal(len(device.trainnet.hidden["fw0"].params), 2)
assert_in("out1", device.trainnet.output)
assert_equal(len(device.trainnet.output), 2)
assert_in("W_in_fw0_out1", device.trainnet.output["out1"].params)
assert_in("b_out1", device.trainnet.output["out1"].params)
assert_equal(len(device.trainnet.output["out1"].params), 2)
assert_in(device.trainnet.hidden["fw0"].params["W_in_data_fw0"], update_dict)
assert_in(device.trainnet.hidden["fw0"].params["b_fw0"], update_dict)
assert_in(device.trainnet.output["out1"].params["W_in_fw0_out1"], update_dict)
assert_in(device.trainnet.output["out1"].params["b_out1"], update_dict)
assert_in(device.trainnet.output["out2"].params["W_in_fw0_out2"], update_dict)
assert_in(device.trainnet.output["out2"].params["b_out2"], update_dict)
assert_equal(len(update_dict), 6)
# Set net params.
net_params = {
"fw0": {"W_in_data_fw0": numpy.identity(3, dtype="float32"),
"b_fw0": numpy.zeros((3,), dtype="float32")},
"out1": {"W_in_fw0_out1": numpy.arange(0.0, 1.2, 0.1, dtype="float32").reshape((3, 4)),
"b_out1": numpy.arange(0.0, 4, dtype="float32")},
"out2": {"W_in_fw0_out2": numpy.arange(0.0, 1.5, 0.1, dtype="float32").reshape((3, 5)),
"b_out2": numpy.arange(0.0, 5, dtype="float32")}
}
device.trainnet.set_params_by_dict(net_params)
device.testnet.set_params_by_dict(net_params)
# Show params.
for p in param_vars:
print "init %s:" % p
pprint(p.get_value())
# Init dataset.
dataset = StaticDataset(data=[{
"data": numpy.array([[0.1, 0.2, -0.3]], dtype="float32"),
"t1": numpy.array([2]),
"t2": numpy.array([4])
}], output_dim=config.typed_value("num_outputs"))
dataset.init_seq_order()
assert_equal(dataset.is_data_sparse("data"), False)
assert_equal(dataset.is_data_sparse("t1"), True)
assert_equal(dataset.is_data_sparse("t2"), True)
# Copy to device allocation.
success = assign_dev_data_single_seq(device, dataset, 0)
assert_true(success, "failed to allocate & assign data")
# Check allocated data.
assert_equal(device.targets["data"].shape, (1, 1, 3)) # input shape. (time,batch,dim)
assert_in("t1", device.targets)
assert_in("t2", device.targets)
assert_equal(device.targets["t1"].shape, (1, 1))
assert_equal(device.targets["t2"].shape, (1, 1))
assert_equal(device.output_index["data"].shape, (1, 1))
numpy.testing.assert_equal(device.output_index["data"], numpy.array([[1]]))
assert_equal(device.output_index["t1"].shape, (1, 1))
numpy.testing.assert_equal(device.output_index["t1"], numpy.array([[1]]))
# Forward test.
device.update_data()
device.testnet.costs["out1"].name = "out1_cost" # nice in the func graph
out_i1 = device.testnet.output["out1"].index
out_i1_nonzero = device.testnet.output["out1"].i
nll1, pcx1 = T.nnet.crossentropy_softmax_1hot(x=device.testnet.output["out1"].y_m[out_i1_nonzero],
y_idx=device.testnet.output["out1"].y_data_flat[out_i1_nonzero])
forward_func = theano.function(
inputs=[device.block_start, device.block_end],
outputs=[
device.testnet.j["t1"], out_i1, out_i1_nonzero[0], nll1, pcx1,
device.testnet.costs["out1"],
device.testnet.output["out1"].p_y_given_x,
device.testnet.costs["out2"],
device.testnet.output["out2"].p_y_given_x],
givens=device.make_givens(device.testnet),
no_default_updates=True,
on_unused_input='warn',
name="forward")
#print "forward func:"
#theano.printing.debugprint(forward_func)
net_j1, out_i1_val, out_i1_nz_val, nll1_val, pcx1_val, t1_cost, t1_y, t2_cost, t2_y = forward_func(0, 1)
print "forward results:"
pprint(net_j1)
pprint(out_i1_val)
pprint(out_i1_nz_val)
pprint(nll1_val)
pprint(pcx1_val)
pprint(t1_cost)
pprint(t1_y)
pprint(t2_cost)
pprint(t2_y)
assert_equal(net_j1, numpy.array([[1]]))
assert_equal(out_i1_val, numpy.array([[1]]))
assert_equal(out_i1_nz_val, numpy.array([0]))
assert_almost_equal(nll1_val, numpy.array([t1_cost]))
numpy.testing.assert_almost_equal(t1_y, pcx1_val)
assert_almost_equal(t1_cost, 1.440189698561195, places=6)
assert_almost_equal(t2_cost, 0.45191439593759336, places=6)
numpy.testing.assert_almost_equal(t1_y, numpy.array([[ 0.0320586 , 0.08714432, 0.23688282, 0.64391426]]), decimal=6)
numpy.testing.assert_almost_equal(t2_y, numpy.array([[ 0.01165623, 0.03168492, 0.08612854, 0.23412166, 0.63640865]]), decimal=6)
# One train step.
device.set_learning_rate(config.typed_value("learning_rate"))
device.run("train")
output_list, outputs_format = device.result()
assert_is_instance(output_list, list)
assert_true(outputs_format, "for train, we should always get the format")
outputs = Device.make_result_dict(output_list, outputs_format)
pprint(outputs)
assert_in("cost:out1", outputs)
assert_greater(outputs["cost:out1"], 0)
assert_almost_equal(outputs["cost:out1"], t1_cost)
# Get net params.
params = device.get_net_train_params(device.trainnet)
references_params = {
"W_in_data_fw0":
numpy.array([[ 1.00055406e+00, 5.54056978e-04, 5.54056978e-04],
[ 1.10811396e-03, 1.00110811e+00, 1.10811396e-03],
[ -1.66217093e-03, -1.66217093e-03, 9.98337829e-01]]),
"b_fw0":
numpy.array([ 0.00554057, 0.00554057, 0.00554057]),
"W_in_fw0_out1":
numpy.array([[-0.00320586, 0.09128557, 0.27631172, 0.23560857],
[ 0.39358828, 0.48257114, 0.75262344, 0.57121715],
[ 0.80961758, 0.9261433 , 0.77106485, 1.29317428]]),
"b_out1":
numpy.array([-0.0320586 , 0.91285568, 2.76311718, 2.35608574]),
"W_in_fw0_out2":
numpy.array([[ -1.16562310e-03, 9.68315079e-02, 1.91387146e-01,
2.76587834e-01, 4.36359135e-01],
[ 4.97668754e-01, 5.93663016e-01, 6.82774291e-01,
7.53175669e-01, 9.72718271e-01],
[ 1.00349687e+00, 1.10950548e+00, 1.22583856e+00,
1.37023650e+00, 1.29092259e+00]]),
"b_out2":
numpy.array([-0.01165623, 0.96831508, 1.91387146, 2.76587834, 4.36359135])
}
assert_equal(len(param_vars), len(params))
for p, v in zip(param_vars, params):
print "%s:" % p
pprint(v)
assert_true(p.name)
numpy.testing.assert_almost_equal(references_params[p.name], v, decimal=6)
def test_combi_auto_enc_longer():
config = Config()
config.update({
"multiprocessing": False,
"blocking": True,
"device": "cpu",
"num_epochs": 1,
"num_inputs": 3,
"num_outputs": {"classes": 2},
"learning_rate": 1.0,
"adadelta": True,
"network": {
"output": {"class": "softmax", "loss": "ce", "target": "classes"},
"auto-enc": {"class": "softmax", "loss": "sse", "dtype": "float32", "target": "data"}
}
})
device = Device("cpu", config=config, blocking=True)
# Set net params.
def get_net_params(with_auto_enc=True):
d = {
"output": {"W_in_data_output": numpy.arange(0.1, 0.7, 0.1, dtype="float32").reshape((3, 2)),
"b_output": numpy.arange(0.0, 2, dtype="float32")}
}
if with_auto_enc:
d["auto-enc"] = {"W_in_data_auto-enc": numpy.arange(0.1, 1.0, 0.1, dtype="float32").reshape((3, 3)),
"b_auto-enc": numpy.arange(0.0, 3, dtype="float32")}
return d
device.trainnet.set_params_by_dict(get_net_params())
device.testnet.set_params_by_dict(get_net_params())
# Show params.
for p in device.trainnet.get_all_params_vars():
print "init %s:" % p
pprint(p.get_value())
# Init dataset.
dataset = DummyDataset(input_dim=config.typed_value("num_inputs"),
output_dim=config.typed_value("num_outputs"),
num_seqs=10)
dataset.init_seq_order()
cost_output_sum = 0.0
for seq_idx in range(dataset.num_seqs):
# Copy to device allocation.
success = assign_dev_data_single_seq(device, dataset, seq_idx)
assert_true(success, "failed to allocate & assign data")
# One train step.
device.set_learning_rate(config.typed_value("learning_rate"))
device.run("train")
output_list, outputs_format = device.result()
assert_is_instance(output_list, list)
assert_true(outputs_format, "for train, we should always get the format")
outputs = Device.make_result_dict(output_list, outputs_format)
print("seq %i" % seq_idx)
pprint(outputs)
assert_in("cost:output", outputs)
assert_in("cost:auto-enc", outputs)
cost_output_sum += outputs["cost:output"]
# Now, drop the auto-enc from the network, and redo the same thing.
del config.typed_value("network")["auto-enc"]
device = Device("cpu", config=config, blocking=True)
device.trainnet.set_params_by_dict(get_net_params(with_auto_enc=False))
device.testnet.set_params_by_dict(get_net_params(with_auto_enc=False))
for p in device.trainnet.get_all_params_vars():
print "second run, init %s:" % p
pprint(p.get_value())
dataset.init_seq_order() # reset
cost2_output_sum = 0.0
for seq_idx in range(dataset.num_seqs):
# Copy to device allocation.
success = assign_dev_data_single_seq(device, dataset, seq_idx)
assert_true(success, "failed to allocate & assign data")
# One train step.
device.set_learning_rate(config.typed_value("learning_rate"))
device.run("train")
output_list, outputs_format = device.result()
assert_is_instance(output_list, list)
assert_true(outputs_format, "for train, we should always get the format")
outputs = Device.make_result_dict(output_list, outputs_format)
print("seq %i" % seq_idx)
pprint(outputs)
assert_in("cost:output", outputs)
assert_not_in("cost:auto-enc", outputs)
cost2_output_sum += outputs["cost:output"]
assert_equal(cost_output_sum, cost2_output_sum)
assert_almost_equal(cost_output_sum, 16.028842568397522, places=6)
def test_combi_auto_enc():
config = Config()
config.update({
"multiprocessing": False,
"blocking": True,
"device": "cpu",
"num_epochs": 1,
"num_inputs": 3,
"num_outputs": {"classes": 2},
"learning_rate": 1.0,
"network": {
"output": {"class": "softmax", "loss": "ce", "target": "classes"},
"auto-enc": {"class": "softmax", "loss": "sse", "dtype": "float32", "target": "data"}
}
})
device = Device("cpu", config=config, blocking=True)
# Set net params.
def get_net_params(with_auto_enc=True):
d = {
"output": {"W_in_data_output": numpy.arange(0.1, 0.7, 0.1, dtype="float32").reshape((3, 2)),
"b_output": numpy.arange(0.0, 2, dtype="float32")}
}
if with_auto_enc:
d["auto-enc"] = {"W_in_data_auto-enc": numpy.arange(0.1, 1.0, 0.1, dtype="float32").reshape((3, 3)),
"b_auto-enc": numpy.arange(0.0, 3, dtype="float32")}
return d
device.trainnet.set_params_by_dict(get_net_params())
device.testnet.set_params_by_dict(get_net_params())
# Show params.
for p in device.trainnet.get_all_params_vars():
print "init %s:" % p
pprint(p.get_value())
# Init dataset.
dataset = StaticDataset(data=[{
"data": numpy.array([[0.1, 0.2, -0.3]], dtype="float32"),
"classes": numpy.array([1]),
}], output_dim=config.typed_value("num_outputs"))
dataset.init_seq_order()
# Copy to device allocation.
success = assign_dev_data_single_seq(device, dataset, 0)
assert_true(success, "failed to allocate & assign data")
# One train step.
device.set_learning_rate(config.typed_value("learning_rate"))
device.run("train")
output_list, outputs_format = device.result()
assert_is_instance(output_list, list)
assert_true(outputs_format, "for train, we should always get the format")
outputs = Device.make_result_dict(output_list, outputs_format)
pprint(outputs)
assert_in("cost:output", outputs)
assert_in("cost:auto-enc", outputs)
expected_cost_output = 0.3132616877555847
assert_almost_equal(outputs["cost:output"], expected_cost_output, places=6)
exact_cost_output = outputs["cost:output"]
assert_almost_equal(outputs["cost:auto-enc"], 5.263200283050537, places=6)
# Now, drop the auto-enc from the network, and redo the same thing.
del config.typed_value("network")["auto-enc"]
device = Device("cpu", config=config, blocking=True)
device.trainnet.set_params_by_dict(get_net_params(with_auto_enc=False))
device.testnet.set_params_by_dict(get_net_params(with_auto_enc=False))
for p in device.trainnet.get_all_params_vars():
print "second run, init %s:" % p
pprint(p.get_value())
dataset.init_seq_order() # reset. probably not needed
success = assign_dev_data_single_seq(device, dataset, 0)
assert_true(success, "failed to allocate & assign data")
device.set_learning_rate(config.typed_value("learning_rate"))
device.run("train")
output_list, outputs_format = device.result()
assert_is_instance(output_list, list)
assert_true(outputs_format, "for train, we should always get the format")
outputs = Device.make_result_dict(output_list, outputs_format)
pprint(outputs)
assert_in("cost:output", outputs)
assert_not_in("cost:auto-enc", outputs)
assert_almost_equal(outputs["cost:output"], expected_cost_output, places=6)
assert_equal(outputs["cost:output"], exact_cost_output)
def _forward(segment_name, features):
"""
:param numpy.ndarray features: format (input-feature,time) (via Sprint)
:return: format (output-dim,time)
:rtype: numpy.ndarray
"""
print("Sprint forward", segment_name, features.shape)
start_time = time.time()
assert engine is not None, "not initialized"
assert sprintDataset
# Features are in Sprint format (feature,time).
num_time = features.shape[1]
assert features.shape == (InputDim, num_time)
time_a = time.time()
dataset, seq_idx = features_to_dataset(features=features,
segment_name=segment_name)
time_b = time.time()
if BackendEngine.is_theano_selected():
# Prepare data for device.
device = engine.devices[0]
success = assign_dev_data_single_seq(device,
dataset=dataset,
seq=seq_idx)
assert success, "failed to allocate & assign data for seq %i, %s" % (
seq_idx, segment_name)
time_c = time.time()
# Do the actual forwarding and collect result.
device.run("extract")
result, _ = device.result()
assert result is not None, "Device crashed."
assert len(result) == 1
posteriors = result[0]
elif BackendEngine.is_tensorflow_selected():
posteriors = engine.forward_single(dataset=dataset, seq_idx=seq_idx)
else:
raise NotImplementedError("unknown backend engine")
time_d = time.time()
# If we have a sequence training criterion, posteriors might be in format (time,seq|batch,emission).
if posteriors.ndim == 3:
assert posteriors.shape == (num_time, 1, OutputDim * MaxSegmentLength)
posteriors = posteriors[:, 0]
# Posteriors are in format (time,emission).
assert posteriors.shape == (num_time, OutputDim * MaxSegmentLength)
# Reformat to Sprint expected format (emission,time).
posteriors = posteriors.transpose()
assert posteriors.shape == (OutputDim * MaxSegmentLength, num_time)
stats = (numpy.min(posteriors), numpy.max(posteriors),
numpy.mean(posteriors), numpy.std(posteriors))
print("posteriors min/max/mean/std:", stats, "time:",
time.time() - start_time,
time.time() - time_a,
time.time() - time_b,
time.time() - time_c,
time.time() - time_d)
if numpy.isinf(posteriors).any() or numpy.isnan(posteriors).any():
print("posteriors:", posteriors)
debug_feat_fn = "/tmp/crnn.pid%i.sprintinterface.debug.features.txt" % os.getpid(
)
debug_post_fn = "/tmp/crnn.pid%i.sprintinterface.debug.posteriors.txt" % os.getpid(
)
print("Wrote to files %s, %s" % (debug_feat_fn, debug_post_fn))
numpy.savetxt(debug_feat_fn, features)
numpy.savetxt(debug_post_fn, posteriors)
assert False, "Error, posteriors contain invalid numbers."
return posteriors