def test_dynamic_concat():
seq1 = [[1, 2], [3, 4]]
seq2 = [[1, 2, 3], [4, 5, 6]]
n = 10
m = 4
inputs = tx.Input(seq2, shape=[None, None], dtype=tf.int32, constant=False)
inputs2 = tx.Input(seq2, dtype=tf.int32, constant=True)
lookup = tx.Lookup(inputs, seq_size=None, embedding_shape=[n, m])
lookup2 = tx.Lookup(inputs2, seq_size=3, embedding_shape=[n, m])
concat1 = lookup.as_concat()
concat2 = lookup2.as_concat()
assert concat1.n_units is None
assert concat2.n_units is not None
concat3 = tx.SeqConcat(lookup, time_major=False)
concat4 = tx.SeqConcat(lookup, seq_size=3, time_major=False)
assert tx.shape_equal(concat4.shape, (None, 3 * 4))
c1, c2 = concat1(), concat3()
assert tx.tensor_equal(c1, c2)
assert concat3.n_units is None
assert concat4.n_units == 3 * lookup.n_units
inputs.value = seq1
l1 = lookup()
inputs.value = seq2
l2 = lookup()
assert np.shape(l1)[-1] == m
assert np.shape(l2)[-1] == m
def test_module_rnn():
""" Module + RNN integration
"""
# test wrapping module around RNN because it has input dependencies that might not be given in the constructor
x1 = tx.Input(tf.ones([1, 2, 3]), n_units=3, name="x1")
x2 = tx.Input(tf.ones([1, 2, 3]), n_units=3, name="x2")
rnn1 = tx.RNN(x1,
cell_config=tx.LSTMCell.config(n_units=4),
n_units=4,
stateful=False)
rnn2 = tx.RNN(x1,
cell_config=tx.LSTMCell.config(n_units=4),
n_units=4,
stateful=False)
out = tx.Concat(rnn1, rnn2)
# add previous state as a dependency to a module
m = tx.Module(inputs=x1,
output=out,
dependencies=rnn1.previous_state + rnn2.previous_state)
m2 = m.reuse_with(x2)
var_layers = set()
for node in m2.graph.dependency_iter():
if isinstance(node, tx.VariableLayer):
var_layers.add(node)
assert var_layers == set(rnn1.previous_state + rnn2.previous_state)
assert tx.tensor_equal(m(), m2())
def test_model_run():
data1 = tf.constant([[1., 1.]])
x = tx.Input(n_units=2, name="x", constant=False)
labels = tx.Input(n_units=2, name="y_", constant=False)
y = tx.Linear(x, 2, name="y")
out1 = tx.Activation(y, tf.nn.softmax)
out2 = tx.Activation(y, tf.nn.softmax)
@tx.layer(n_units=2, name="loss")
def loss(pred, labs):
return tf.losses.categorical_crossentropy(labs, pred)
model = tx.Model(run_inputs=x,
run_outputs=[out1, out2],
train_inputs=[x, labels],
train_outputs=out1,
train_loss=loss(out1, labels))
model.set_optimizer(tf.optimizers.SGD, lr=0.5)
result1 = model.run({x: data1})
result2 = model.run([data1])
assert tx.tensor_equal(result1[0], result2[0])
assert tx.tensor_equal(result1[1], result2[1])
result3 = model.run({x: data1}, compiled_graph=True)
assert tx.tensor_equal(result3[0], result2[0])
assert tx.tensor_equal(result3[1], result2[1])
def test_input_value():
inputs = tx.Input(n_units=4, dtype=tf.int32, constant=False)
assert tx.tensor_equal(inputs.value, tf.zeros([1, 4], dtype=tf.int32))
with pytest.raises(ValueError):
inputs.value = np.ones([2, 3], dtype=np.int32)
inputs.value = np.ones([2, 4], dtype=np.int32)
assert inputs.value is not None
assert inputs() is not None
assert inputs().dtype == tf.int32
# test sparse input
inputs = tx.Input(n_units=4, n_active=2, dtype=tf.int64, constant=False)
assert tx.tensor_equal(inputs.value, tf.zeros([0, 2], dtype=tf.int64))
with pytest.raises(ValueError) as ve:
inputs.value = [[0, 2, 2]]
assert "Invalid shape" in str(ve)
inputs.value = [[0, 2]]
# create an equivalent sparse input
sp_input = inputs()
assert isinstance(sp_input, tf.SparseTensor)
inputs2 = tx.Input(n_units=4, init_value=sp_input)
dense_value = tf.sparse.to_dense(inputs())
dense_value2 = tf.sparse.to_dense(inputs2())
expected = tf.constant([[1, 0, 1, 0]], dtype=np.int64)
assert tx.tensor_equal(expected, dense_value)
assert tx.tensor_equal(dense_value, dense_value2)
def test_loss_model_dependencies():
inputs = tx.Input(n_units=2, name="x", constant=False)
labels = tx.Input(n_units=2, name="y_", constant=False)
y = tx.Linear(inputs, 2, name="y")
out1 = tx.Activation(y, tf.nn.softmax, name="out1")
out2 = tx.Activation(y, tf.nn.softmax, name="out2")
@tx.layer(n_units=2, name="loss")
def loss(pred, labs):
return tf.losses.categorical_crossentropy(labs, pred)
logging.basicConfig(level=logging.DEBUG)
model = tx.Model(run_inputs=inputs,
run_outputs=[out1, out2],
train_inputs=[inputs, labels],
train_outputs=[out2, out1],
train_loss=loss(out1, labels))
lr = tx.Param(0.5)
opt = model.set_optimizer(tf.optimizers.SGD, lr=lr)
assert isinstance(opt, tf.optimizers.Optimizer)
it = model.train_graph.dependency_iter()
layers = list(it)
assert layers[0] is inputs
assert layers[1] is labels
assert len(layers) == 6
def test_layer_graph():
data = [[1., 2.]]
in1 = tx.Input(n_units=2, name="in1", constant=False)
in2 = tx.Input(n_units=2, name="in2", constant=False)
linear = tx.Linear(in1, 1, add_bias=False)
graph = tx.Graph.build(inputs=in1, outputs=linear)
assert in1 in graph.in_nodes
with pytest.raises(ValueError):
tx.Graph.build(inputs=[in1, in2], outputs=linear)
pytest.fail(
"Expected ValueError: some inputs are not connected to anything")
with pytest.raises(ValueError):
tx.Graph.build(inputs=[in2], outputs=linear)
pytest.fail(
"Expected ValueError: inputs specified but dependencies are missing"
)
w = tf.matmul(data, linear.weights)
in1.value = data
r1 = linear()
r2 = graph(data)
assert tx.tensor_equal(r2[0], w)
assert tx.tensor_equal(r1, w)
def test_set_optimizer():
x = tx.Input(n_units=2, name="x", constant=False)
labels = tx.Input(n_units=2, name="labels", constant=False)
y = tx.Linear(x, 2, name="y")
out1 = tx.Activation(y, tf.nn.softmax)
out2 = tx.Activation(y, tf.nn.softmax)
@tx.layer(n_units=2, name="loss")
def loss(pred, labs):
return tf.losses.categorical_crossentropy(labs, pred)
model = tx.Model(run_inputs=x,
run_outputs=[out1, out2],
train_inputs=[x, labels],
train_outputs=[out2, out1],
train_loss=loss(out1, labels))
lr = tx.Param(0.5)
opt = model.set_optimizer(tf.optimizers.SGD,
learning_rate=lr,
clipnorm=0.1)
assert isinstance(opt, tf.optimizers.Optimizer)
assert model.optimizer.get_config()["learning_rate"] == 0.5
data1 = [[1., 1.], [1., 1.]]
data2 = tf.constant([[0., 1.], [0., 1.]])
params = model.optimizer_params[model.optimizer]
data_dict, params_dict = tx.Model.parse_input(
{
x: data1,
"learning_rate": 0.2
}, model.run_graph.in_nodes, params)
assert len(data_dict) == 1
assert len(params_dict) == 1
assert model.optimizer_params[opt]["learning_rate"] is lr
result1 = model.train_step({x: data1, labels: data2})
result2 = model.train_step([data1, data2])
assert len(result1) == 3
assert len(result2) == 3
assert tf.reduce_all(tf.less(result2[-1], result1[-1]))
result1 = model.run({x: np.array(data1, dtype=np.float32)})
result2 = model.run([data1])
result3 = model.run(np.array(data1, np.float32))
x.value = data1
o2 = out2()
o1 = out1()
result4 = (o2, o1)
for i in range(2):
assert tx.tensor_equal(result1[i], result2[i])
assert tx.tensor_equal(result1[i], result3[i])
assert tx.tensor_equal(result1[i], result4[i])
def test_merge_add_shape():
x1 = tx.Input([[2.]], n_units=1, name="x1")
x2 = tx.Input([[2.]], n_units=1, name="x2")
add = tx.Add(x1, x2)
assert len(add.shape) == 2
assert add.shape[-1] == 1
assert add.shape[0] is None
def test_rnn_cell_drop():
n_hidden = 4
inputs1 = tx.Input(np.ones([2, 100]), dtype=tf.float32)
inputs2 = tx.Input(np.ones([2, 100]), dtype=tf.float32)
with tf.name_scope("wtf"):
rnn1 = tx.RNNCell(inputs1,
n_hidden,
x_dropout=0.5,
r_dropout=0.5,
u_dropconnect=0.5,
w_dropconnect=0.5,
regularized=True)
rnn2 = rnn1.reuse_with(inputs2, rnn1)
rnn3 = rnn1.reuse_with(inputs2, rnn1)
rnn4 = rnn1.reuse_with(inputs2, None, regularized=False)
rnn5 = rnn4.reuse_with(inputs2, None, regularized=True)
r1, r2, r3, r4, r5 = rnn1(), rnn2(), rnn3(), rnn4(), rnn5()
# w is a linear layer from the input but a regularized layer applies dropout to the input, so we have a dropout
# in between
# without a shared state object, we couldn't rewire graphs, in the case of non-eager we can share a tensor
# that is already wired with something (it takes the shape of the input of one layer and creates a mask tensor
# shared across dropout instances
# Linear layers should have shared states as well, in this case sharing the weights
# dropout_state1 = rnn1.w.input_layers[0].layer_state
# dropout_state2 = rnn2.w.input_layers[0].layer_state
# dropout_state3 = rnn3.w.input_layers[0].layer_state
# mask1, mask2, mask3 = dropout_state1.mask, dropout_state2.mask, dropout_state3
assert tx.tensor_equal(r2, r3)
assert not tx.tensor_equal(r2, r4)
assert not tx.tensor_equal(r4, r5)
assert rnn1.dropout_locked
assert rnn2.dropout_locked
assert hasattr(rnn1, "w")
assert hasattr(rnn2, "w")
w1: tx.Layer = getattr(rnn1, "w")
w2: tx.Layer = getattr(rnn2, "w")
assert isinstance(w1, tx.DropConnect)
state1, state2 = w1.layer_state, w2.layer_state
assert hasattr(state1, "weight_mask")
assert hasattr(state2, "weight_mask")
# dropout locked == true
mask1 = getattr(state1, "weight_mask")
mask2 = getattr(state2, "weight_mask")
assert tx.tensor_equal(mask1, mask2)
def test_reshape_shape():
x = tf.reshape(tf.range(9), [3, 3, 1])
x = tx.Input(x, dtype=tf.float32)
flat = tx.Reshape(x, [-1, 1])
assert flat.shape[0] is None
assert flat.shape[-1] == 1
x = tx.Input(x, shape=[3, 3, 1], dtype=tf.float32)
print(x.shape)
flat = tx.Reshape(x, [-1, 1])
print(flat.shape)
def test_build_graph():
x1 = tx.Input(n_units=1000, constant=False, dtype=tf.float32)
x2 = tx.Input(init_value=tf.ones([1, 3]), dtype=tf.float32, constant=True)
y10 = tx.Linear(x1, n_units=3)
y11 = tx.Activation(y10)
y1 = tx.Module(x1, y11)
y2 = tx.Add(y1, x2)
output = y2
graph = Graph.build(inputs=None, outputs=[y1, y2])
# module condenses 2 nodes so it's 4 and not 6
assert len(graph.nodes) == 4
@tf.function
def simple_graph(in0):
x1.value = in0
return y2()
simple_graph_2 = Graph.build(inputs=[x1, x2], outputs=y2)
simple_graph_2 = tf.function(simple_graph_2)
g = Graph.build(inputs=[x1, x2], outputs=y2)
y2fn = y2.as_function()
data = tf.ones([256, 1000])
x1.value = data
compiled_fn = g.as_function(ord_inputs=x1, ord_outputs=output)
assert tx.tensor_equal(compiled_fn(data), y2fn())
assert tx.tensor_equal(compiled_fn(data), simple_graph_2()[0])
from timeit import timeit
def update_run():
x1.value = tf.random.uniform([256, 1000])
return y2fn()
n = 1000
t_update_run = timeit(update_run, number=n)
t_generated = timeit(lambda: compiled_fn(tf.random.uniform([256, 1000])),
number=n)
t_compile_value_set = timeit(
lambda: simple_graph(tf.random.uniform([256, 1000])), number=n)
t_graph_call_tf = timeit(
lambda: simple_graph_2(tf.random.uniform([256, 1000])), number=n)
assert t_generated < t_update_run
assert t_generated < t_compile_value_set
assert t_generated < t_graph_call_tf
assert t_update_run > t_compile_value_set
o1 = compiled_fn(tf.random.uniform([256, 1000]))
o2 = compiled_fn(tf.random.uniform([256, 1000]))
assert not tx.tensor_equal(o1, o2)
def test_nnlm_lstm(self):
vocab_size = 10000
ctx_size = 5
batch_size = 20
embed_dim = 64
h_dim = 128
print('building model')
inputs = tx.Input(n_units=None, dtype=tf.int64, name="ctx_inputs")
labels = tx.Input(n_units=None, dtype=tf.int64, name="labels")
model = NNLM_LSTM(inputs=inputs,
labels=labels,
vocab_size=vocab_size,
embed_dim=embed_dim,
embed_share=True,
use_f_predict=True,
h_dim=h_dim,
num_h=2,
embed_dropout=0.3,
w_dropconnect=None,
u_dropconnect=None,
y_dropout=0.3,
r_dropout=0.3,
other_dropout=0.3,
use_nce=False,
nce_samples=2)
model.run_graph.draw("run.pdf")
print("done")
model.set_session(runtime_stats=True)
model.set_log_dir("/tmp/")
model.log_graph()
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
# options = None
model.set_session(runtime_stats=True, run_options=options)
model.config_optimizer(
tf.train.AdamOptimizer(learning_rate=0.005),
gradient_op=lambda grad: tf.clip_by_norm(grad, 4.0))
# model.train_graph.draw("train.pdf")
input_data = np.random.randint(0, vocab_size, [batch_size, ctx_size])
label_data = np.random.randint(0, vocab_size, [batch_size, ctx_size])
print(np.shape(label_data))
with self.cached_session():
for i in tqdm(range(100)):
if i == 50:
model.reset_state()
#eval1 = model.eval({inputs: input_data, labels: label_data})
#eval2 = model.eval({inputs: input_data, labels: label_data})
result = model.train({inputs: input_data, labels: label_data})
def test_graph_draw(tmpdir):
x = tx.Input([[1]])
x2 = tx.Input([[1]])
l1 = tx.Linear(x, 2, name="l1")
l2 = tx.Linear(x, 2, name="l2")
l3 = tx.layer(n_units=2, name="l3")(lambda a, b: tf.add(a, b))(l1, l2)
l4 = l2.reuse_with(x2)
graph = Graph.build(inputs=[x, x2], outputs=[l3, l4])
str_path = str(tmpdir.join("test.pdf"))
graph.draw(path=str_path)
assert os.path.exists(str_path)
def test_graph_input_order():
in1 = tx.Input(n_units=1, name="in1", dtype=tf.float32, constant=False)
in2 = tx.Input(n_units=1, name="in2", dtype=tf.float32, constant=False)
in12 = tx.Add(in1, in2)
in3 = tx.Constant(tf.ones(shape=[1], dtype=tf.float32))
in123 = tx.Add(in12, in3)
graph = tx.Graph.build(inputs=None, outputs=in123)
# print("\n")
# for layer,p in graph.dependency_iter().items():
# print(layer.name)
# print(p)
print(list(map(lambda x: x.name, graph.in_nodes)))
def test_nnlm(self):
vocab_size = 4
ctx_size = 22
batch_size = 2
embed_dim = 512
h_dim = 128
inputs = tx.Input(ctx_size, dtype=tf.int64, name="ctx_inputs")
labels = tx.Input(1, dtype=tf.int64, name="ctx_inputs")
model = NNLM(inputs=inputs,
label_inputs=labels,
vocab_size=vocab_size,
embed_dim=embed_dim,
embed_share=True,
use_f_predict=True,
h_dim=h_dim,
use_dropout=False,
drop_probability=0.9,
embed_dropout=False,
use_nce=True,
nce_samples=2)
model.set_session(runtime_stats=True)
model.set_log_dir("/tmp/")
model.log_graph()
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
# options = None
model.set_session(runtime_stats=True, run_options=options)
model.config_optimizer(
tf.train.AdamOptimizer(learning_rate=0.005),
gradient_op=lambda grad: tf.clip_by_norm(grad, 4.0))
input_data = np.random.randint(0, vocab_size, [batch_size, ctx_size])
label_data = np.random.randint(0, vocab_size, [batch_size, 1])
with self.cached_session():
for _ in tqdm(range(1)):
eval1 = model.eval({inputs: input_data, labels: label_data})
eval2 = model.eval({inputs: input_data, labels: label_data})
result = model.train({
inputs: input_data,
labels: label_data
},
write_summaries=True)
# print(list(map(str,model.train_graph.output_layers)))
self.assertArrayEqual(eval1, eval2)
self.assertArrayNotEqual(result, eval2)
def test_rnn_cell():
n_inputs = 3
n_units = 4
batch_size = 2
inputs = tx.Input(n_units=n_inputs)
rnn0 = tx.RNNCell(inputs, n_units)
# Keras RNN cell
rnn1 = SimpleRNNCell(n_units)
state = rnn1.get_initial_state(inputs, batch_size=1)
assert tx.tensor_equal(state, rnn0.previous_state[0]())
inputs.value = tf.ones([batch_size, n_inputs])
res1 = rnn1(inputs, (state, ))
rnn1.kernel = rnn0.layer_state.w.weights
rnn1.bias = rnn0.layer_state.w.bias
rnn1.recurrent_kernel = rnn0.layer_state.u.weights
res2 = rnn1(inputs, (state, ))
assert not tx.tensor_equal(res1[0], res2[0])
assert not tx.tensor_equal(res1[1], res2[1])
res0 = rnn0()
assert tx.tensor_equal(res2[0], res0)
def test_linear_rank3():
val = tf.constant([[[1], [1]], [[2], [2]]])
x1 = tx.Input(val, dtype=tf.float32)
x2 = tx.Transpose(x1)
assert val.shape[1:] == x1.shape[1:]
x1_flat = tx.Reshape(x1, [-1, 1])
linear1 = tx.Linear(x1, n_units=2)
linear2 = tx.Linear(x2,
weights_shape=[2, 1],
weights=linear1.weights,
transpose_weights=True)
# we cant do this because it changes the definition
# of the layer (n_units etc)
with pytest.raises(ValueError):
linear1.reuse_with(x2, transpose_weights=True)
pytest.fail(
"can't reuse with transpose weights while changing the layer definition"
)
linear_flat = linear1.reuse_with(x1_flat, shape=(4, 2))
x1_tensor = x1()
new_shape = x1_tensor.shape[:-1] + [2]
linear_flat = tx.Reshape(linear_flat, new_shape)
assert tx.tensor_equal(linear1(), linear_flat())
assert tx.tensor_equal(tf.shape(linear2()), [1, 2, 1])
def test_lstm_cell():
n_inputs = 4
n_hidden = 2
batch_size = 2
inputs = tx.Input(np.ones([batch_size, n_inputs], np.float32),
n_units=n_inputs,
constant=True)
rnn1 = tx.LSTMCell(inputs, n_hidden, gate_activation=tf.sigmoid)
previous_state = (None, rnn1.state[-1]())
rnn2 = rnn1.reuse_with(inputs, *previous_state)
# if we don't wipe the memory, memory will be reused
previous_state = (None, tx.LSTMCell.zero_state(rnn1.n_units))
rnn3 = rnn1.reuse_with(inputs, *previous_state)
rnn4 = rnn1.reuse_with(inputs)
res1 = rnn1()
res2 = rnn2()
res3 = rnn3()
res4 = rnn4()
assert (batch_size, n_hidden) == np.shape(res1)
assert tx.tensor_equal(res1, res3)
assert not tx.tensor_equal(res1, res2)
assert tx.tensor_equal(res1, res4)
def test_lookup_dynamic_sequence():
seq1 = [[1, 2], [3, 4]]
seq2 = [[1, 2, 3], [4, 5, 6]]
n = 10
h = 4
inputs = tx.Input(dtype=tf.int32, constant=False)
lookup = tx.Lookup(inputs, seq_size=None, embedding_shape=[n, h])
assert tx.shape_equal(lookup.shape, (None, None, h))
concat = lookup.as_concat()
inputs.value = seq1
inputs.value = seq1
inputs()
inputs.value = seq2
inputs()
inputs.value = seq1
l1 = lookup()
inputs.value = seq2
l2 = lookup()
inputs.value = seq1
c1 = concat()
inputs.value = seq2
c2 = concat()
assert np.shape(l1)[-1] == h
assert np.shape(l2)[-1] == h
assert np.shape(c1)[-1] == h * 2
assert np.shape(c2)[-1] == h * 3
def test_coupled_gate():
vocab_size = 4
n_features = 3
seq_size = 2
inputs = tx.Input(init_value=np.array([[2, 0], [1, 2]]),
n_units=seq_size,
dtype=tf.int32,
constant=True)
features1 = tx.Lookup(inputs,
seq_size,
embedding_shape=[vocab_size,
n_features]).as_concat()
features2 = tx.Lookup(inputs,
seq_size,
embedding_shape=[vocab_size,
n_features]).as_concat()
gate_w = tx.Linear(features1, seq_size, add_bias=True)
coupled_gate = tx.CoupledGate(features1, features2, gate_w)
sp_features1 = tx.ToSparse(features1)
assert tx.tensor_equal(tf.sparse.to_dense(sp_features1()), features1())
sp_gate = tx.CoupledGate(sp_features1, features2, gate_w)
print(sp_gate())
print(sp_gate.shape)
# coupled_gate2 = coupled_gate.reuse_with(sp_features1, features2)
r1 = coupled_gate()
def test_rnn_cell():
n_inputs = 4
n_hidden = 2
batch_size = 2
x = tx.Input(init_value=tf.ones([batch_size, n_inputs]), constant=False)
rnn1 = tx.RNNCell(x, n_hidden)
assert rnn1.shape[0] == x.shape[0]
assert rnn1.shape[-1] == rnn1.n_units
state = rnn1.state
state = state[0]()
rnn_2 = rnn1.reuse_with(x, state)
rnn_3 = rnn1.reuse_with(x)
with pytest.raises(TypeError):
tx.RNNCell(x, n_hidden, share_state_with=x)
pytest.fail(
"Type Error Expected: inputs cannot share state with RNNCell")
res1 = rnn1()
res2 = rnn_2()
res3 = rnn_3()
assert (batch_size, n_hidden) == np.shape(res1)
assert tx.tensor_equal(res1, res3)
assert not tx.tensor_equal(res1, res2)
def test_lookup_sequence_transform():
vocab_size = 4
embed_dim = 2
seq_size = 2
inputs = tx.Input(n_units=seq_size, dtype=tf.int32)
input_data = np.array([[2, 0], [1, 2], [0, 2]])
lookup = tx.Lookup(inputs,
seq_size=seq_size,
embedding_shape=[vocab_size, embed_dim],
add_bias=True)
concat_lookup = lookup.as_concat()
seq_lookup = lookup.permute_batch_time()
assert hasattr(lookup, "seq_size")
inputs.value = input_data
v1 = lookup()
v2 = concat_lookup()
v3 = seq_lookup()
assert np.shape(v1) == (np.shape(input_data)[0], seq_size, embed_dim)
assert np.shape(v2) == (np.shape(input_data)[0], seq_size * embed_dim)
assert np.shape(v3) == (seq_size, np.shape(input_data)[0], embed_dim)
assert tx.tensor_equal(v1[:, 0], v3[0])
def test_graph_build():
x = tx.Input([[1]])
g = Graph.build(None, x)
assert len(g.in_nodes) == len(g.out_nodes)
assert len(g.in_nodes) == 1
l1 = tx.Linear(x, n_units=2)
l2 = tx.Linear(x, n_units=2)
l3 = tx.Linear(x, n_units=2)
g1 = Graph.build(None, l1)
assert len(g1.in_nodes) == len(g1.out_nodes)
assert set.isdisjoint(set(g1.in_nodes), g1.out_nodes)
assert l1 in g1.out_nodes
assert x in g1.in_nodes
g2 = Graph.build(x, l1)
assert not set.isdisjoint(set(g1.in_nodes), g2.in_nodes)
assert not set.isdisjoint(set(g1.out_nodes), g2.out_nodes)
with pytest.raises(ValueError):
Graph.build([l2, l3], l1)
pytest.fail("ValueError Expected: invalid graph")
g = Graph.build(x, [l2, l3])
assert len(g.edges_out[x]) == 2
assert l2 in g.edges_out[x]
assert l3 in g.edges_out[x]
assert x == g.edges_in[l2][0]
def test_batch_norm():
v = tf.random.uniform([3, 4])
x = tx.Input(v, dtype=tf.float32)
bn = tx.BatchNorm(x, offset=True, scale=True)
moving_mean = bn.moving_mean
moving_variance = bn.moving_variance
before = bn.moving_mean.value()
bn()
after = bn.moving_mean.value()
assert not tx.tensor_equal(before, after)
x.value = tf.random.uniform([3, 4])
bn = bn.reuse_with(x, training=False)
before = bn.moving_mean.value()
bn()
after = bn.moving_mean.value()
assert tx.tensor_equal(before, after)
assert moving_mean is bn.moving_mean
assert moving_variance is bn.moving_variance
bn = bn.reuse_with(x, training=True)
x.value = tf.random.uniform([3, 4])
before = bn.moving_mean.value()
bn()
after = bn.moving_mean.value()
assert not tx.tensor_equal(before, after)
def test_wrap_shape():
x = tx.Input(n_units=3)
t = tx.Transpose(x, n_units=3)
assert t.shape[-1] == 3
w = tx.Wrap(t, wrap_fn=lambda layer: layer * 2)
assert w.shape == [3, 3]
def test_add_optimizer():
target = tx.Constant([[1.]])
inputs = tx.Input(n_units=2, name="inputs")
output = tx.Linear(inputs, n_units=1, name="y")
loss = tx.Lambda(target,
output,
fn=tf.nn.softmax_cross_entropy_with_logits,
name="xent")
m = tx.Model(run_outputs=output,
train_inputs=[inputs, target],
train_loss=loss)
lr = tx.Param(init_value=0.2, name="lr")
optimizer1 = m.set_optimizer(tf.optimizers.SGD, lr=lr)
optimizer2: tf.optimizers.Optimizer = m.optimizer
assert optimizer1 == optimizer2
# optimizer is hashable
opt_dict = {optimizer1: 0, optimizer2: 1}
assert optimizer1 in opt_dict
assert opt_dict[optimizer1] == 1
lr.value = 0.3
assert np.float32(0.3) == optimizer1.lr.numpy()
def test_gru_cell():
n_inputs = 3
n_units = 4
batch_size = 1
inputs = tx.Input(n_units=n_inputs)
gru0 = tx.GRUCell(inputs,
n_units,
activation=tf.tanh,
gate_activation=tf.sigmoid)
# applies gate after matrix multiplication and uses
# recurrent biases, this makes it compatible with cuDNN
# implementation
gru1 = GRUCell(n_units,
activation='tanh',
recurrent_activation='sigmoid',
reset_after=False,
implementation=1,
use_bias=True)
assert not hasattr(gru1, "kernel")
state0 = [s() for s in gru0.previous_state]
# get_initial_state from keras returns either a tuple or a single
# state see test_rnn_cell, but the __call__ API requires an iterable
state1 = gru1.get_initial_state(inputs, batch_size=1)
assert tx.tensor_equal(state1, state0[0])
inputs.value = tf.ones([batch_size, n_inputs])
res1 = gru1(inputs, state0)
res1_ = gru1(inputs, state0)
for r1, r2 in zip(res1, res1_):
assert tx.tensor_equal(r1, r2)
# the only difference is that keras kernels are fused together
kernel = tf.concat([w.weights.value() for w in gru0.layer_state.w],
axis=-1)
recurrent_kernel = tf.concat([u.weights for u in gru0.layer_state.u],
axis=-1)
bias = tf.concat([w.bias for w in gru0.layer_state.w], axis=-1)
assert tx.same_shape(kernel, gru1.kernel)
assert tx.same_shape(recurrent_kernel, gru1.recurrent_kernel)
assert tx.same_shape(bias, gru1.bias)
gru1.kernel = kernel
gru1.recurrent_kernel = recurrent_kernel
gru1.bias = bias
res2 = gru1(inputs, state0)
for i in range(len(res1)):
assert not tx.tensor_equal(res1[i], res2[i])
res0 = gru0()
# res0_ = gru0.state[0]()
assert tx.tensor_equal(res0, res2[0])
def test_tensor_layer():
x = tx.Input([[2]], n_units=1, constant=False)
fn = x.as_function()
x.value = [[4]]
y = fn()
assert y.numpy().flatten() == 4
def test_to_sparse():
inputs = tx.Input(init_value=tf.ones([2, 100]))
linear = tx.Linear(inputs, n_units=100)
relu = tx.Activation(linear, tx.relu)
sparse = tx.ToSparse(relu)
assert tx.shape_equal(sparse.shape, linear.shape)
assert tx.shape_equal(sparse.shape, relu.shape)
def test_graph_repeated():
x = tx.Input([[1]])
l1 = tx.Linear(x, 2, name="l1")
l2 = tx.Linear(x, 2, name="l2")
l3 = tx.layer(n_units=2, name="l3")(lambda a, b: tf.add(a, b))(l1, l2)
g = Graph.build(l1, l3, add_missing_inputs=True)
assert set([x, l1]) == set(g.in_nodes)