def test_model_run():
data1 = tf.constant([[1., 1.]])
x = tx.Input(n_units=2, name="x", constant=False)
labels = tx.Input(n_units=2, name="y_", constant=False)
y = tx.Linear(x, 2, name="y")
out1 = tx.Activation(y, tf.nn.softmax)
out2 = tx.Activation(y, tf.nn.softmax)
@tx.layer(n_units=2, name="loss")
def loss(pred, labs):
return tf.losses.categorical_crossentropy(labs, pred)
model = tx.Model(run_inputs=x,
run_outputs=[out1, out2],
train_inputs=[x, labels],
train_outputs=out1,
train_loss=loss(out1, labels))
model.set_optimizer(tf.optimizers.SGD, lr=0.5)
result1 = model.run({x: data1})
result2 = model.run([data1])
assert tx.tensor_equal(result1[0], result2[0])
assert tx.tensor_equal(result1[1], result2[1])
result3 = model.run({x: data1}, compiled_graph=True)
assert tx.tensor_equal(result3[0], result2[0])
assert tx.tensor_equal(result3[1], result2[1])
def test_loss_model_dependencies():
inputs = tx.Input(n_units=2, name="x", constant=False)
labels = tx.Input(n_units=2, name="y_", constant=False)
y = tx.Linear(inputs, 2, name="y")
out1 = tx.Activation(y, tf.nn.softmax, name="out1")
out2 = tx.Activation(y, tf.nn.softmax, name="out2")
@tx.layer(n_units=2, name="loss")
def loss(pred, labs):
return tf.losses.categorical_crossentropy(labs, pred)
logging.basicConfig(level=logging.DEBUG)
model = tx.Model(run_inputs=inputs,
run_outputs=[out1, out2],
train_inputs=[inputs, labels],
train_outputs=[out2, out1],
train_loss=loss(out1, labels))
lr = tx.Param(0.5)
opt = model.set_optimizer(tf.optimizers.SGD, lr=lr)
assert isinstance(opt, tf.optimizers.Optimizer)
it = model.train_graph.dependency_iter()
layers = list(it)
assert layers[0] is inputs
assert layers[1] is labels
assert len(layers) == 6
def test_add_optimizer():
target = tx.Constant([[1.]])
inputs = tx.Input(n_units=2, name="inputs")
output = tx.Linear(inputs, n_units=1, name="y")
loss = tx.Lambda(target,
output,
fn=tf.nn.softmax_cross_entropy_with_logits,
name="xent")
m = tx.Model(run_outputs=output,
train_inputs=[inputs, target],
train_loss=loss)
lr = tx.Param(init_value=0.2, name="lr")
optimizer1 = m.set_optimizer(tf.optimizers.SGD, lr=lr)
optimizer2: tf.optimizers.Optimizer = m.optimizer
assert optimizer1 == optimizer2
# optimizer is hashable
opt_dict = {optimizer1: 0, optimizer2: 1}
assert optimizer1 in opt_dict
assert opt_dict[optimizer1] == 1
lr.value = 0.3
assert np.float32(0.3) == optimizer1.lr.numpy()
def test_set_optimizer():
x = tx.Input(n_units=2, name="x", constant=False)
labels = tx.Input(n_units=2, name="labels", constant=False)
y = tx.Linear(x, 2, name="y")
out1 = tx.Activation(y, tf.nn.softmax)
out2 = tx.Activation(y, tf.nn.softmax)
@tx.layer(n_units=2, name="loss")
def loss(pred, labs):
return tf.losses.categorical_crossentropy(labs, pred)
model = tx.Model(run_inputs=x,
run_outputs=[out1, out2],
train_inputs=[x, labels],
train_outputs=[out2, out1],
train_loss=loss(out1, labels))
lr = tx.Param(0.5)
opt = model.set_optimizer(tf.optimizers.SGD,
learning_rate=lr,
clipnorm=0.1)
assert isinstance(opt, tf.optimizers.Optimizer)
assert model.optimizer.get_config()["learning_rate"] == 0.5
data1 = [[1., 1.], [1., 1.]]
data2 = tf.constant([[0., 1.], [0., 1.]])
params = model.optimizer_params[model.optimizer]
data_dict, params_dict = tx.Model.parse_input(
{
x: data1,
"learning_rate": 0.2
}, model.run_graph.in_nodes, params)
assert len(data_dict) == 1
assert len(params_dict) == 1
assert model.optimizer_params[opt]["learning_rate"] is lr
result1 = model.train_step({x: data1, labels: data2})
result2 = model.train_step([data1, data2])
assert len(result1) == 3
assert len(result2) == 3
assert tf.reduce_all(tf.less(result2[-1], result1[-1]))
result1 = model.run({x: np.array(data1, dtype=np.float32)})
result2 = model.run([data1])
result3 = model.run(np.array(data1, np.float32))
x.value = data1
o2 = out2()
o1 = out1()
result4 = (o2, o1)
for i in range(2):
assert tx.tensor_equal(result1[i], result2[i])
assert tx.tensor_equal(result1[i], result3[i])
assert tx.tensor_equal(result1[i], result4[i])
def test_model_vars():
target = tx.Constant([[1.]])
inputs = tx.Input(n_units=2, name="inputs", constant=False)
output = tx.Linear(inputs, n_units=1, name="y")
loss = tx.Lambda(target,
output,
fn=tf.nn.softmax_cross_entropy_with_logits,
name="xent")
m = tx.Model(run_outputs=output,
train_inputs=[inputs, target],
train_loss=loss)
assert m.trainable_variables == output.trainable_variables
def test_model_var_inputs():
# wanted to test when our train graph has more inputs that do not need to be fed (e.g. variable state)
n_features = 5
embed_size = 4
hidden_dim = 3
seq_size = 3
out_size = 2
batch_size = 2
x = tx.Input(np.random.random([batch_size, seq_size]),
n_units=seq_size,
dtype=tf.int32)
y = tx.Input(np.random.random([batch_size, out_size]),
n_units=out_size,
dtype=tf.float32)
lookup = tx.Lookup(x,
seq_size=seq_size,
embedding_shape=[n_features, embed_size])
# seq = lookup.permute_batch_time()
seq = tx.Transpose(lookup, [1, 0, 2])
rnn1 = tx.RNN(seq, cell_config=tx.RNNCell.config(n_units=hidden_dim))
y_ = tx.Linear(rnn1[seq_size - 1], n_units=out_size)
# y_ = tx.Linear(tx.SeqConcat(lookup, seq_size=seq_size), n_units=out_size)
# @tx.layer(n_units=2, dtype=tf.float32, name="loss")
# def loss(pred, labels):
# return tx.mse(pred, labels)
model = tx.Model(run_inputs=x,
run_outputs=y_,
train_inputs=[x, y],
train_outputs=y_,
train_loss=tx.MSE(y_, y))
# model.draw("test.pdf")
model.set_optimizer(tf.optimizers.SGD, lr=0.5)
data1 = [[0, 1, 2], [2, 1, 0]]
data2 = [[0., 1.], [1., 0.]]
model.train_step(input_feed={x: data1, y: data2})
def test_model_train():
x = tx.Input(n_units=2, name="x", constant=False)
labels = tx.Input(n_units=2, name="labels", constant=False)
y = tx.Linear(x, 2, name="y1", add_bias=False)
out1 = tx.Activation(y, tf.nn.softmax)
out2 = tx.Activation(y, tf.nn.softmax)
@tx.layer(n_units=2, name="loss")
def loss(pred, labs):
return tf.losses.categorical_crossentropy(labs, pred)
model = tx.Model(run_inputs=x,
run_outputs=[out1, out2],
train_inputs=[x, labels],
train_outputs=[out2, out1],
train_loss=loss(out1, labels))
lr = tx.Param(0.5)
opt = model.set_optimizer(tf.optimizers.SGD,
learning_rate=lr,
clipnorm=0.1)
data1 = [[1., 1.], [1., 1.]]
data2 = [[0., 1.], [0., 1.]]
w1 = y.weights.numpy()
epochs = 100
model.train(train_data=[{x: data1, labels: data2}], epochs=epochs)
w2 = y.weights.value()
y.weights.assign(w1)
for _ in range(epochs):
model.train_step(input_feed={x: data1, labels: data2})
w3 = y.weights.value()
assert tx.tensor_equal(w2, w3)
data = batch_it(data, batch_size)
label_layer = tx.Input(1)
in_layer = tx.Input(M)
f1 = tx.FC(in_layer, 512, activation=tf.nn.tanh)
f2 = tx.FC(f1, 512, activation=tf.nn.relu)
fm = tx.Highway(f1, f2, carry_gate=True)
out = tx.Linear(f2, 1)
out_prob = tx.Activation(out, fn=tx.sigmoid)
loss = tx.binary_cross_entropy(labels=label_layer.tensor, logits=out.tensor)
model = tx.Model(run_inputs=in_layer,
run_outputs=out_prob,
train_in_loss=label_layer,
train_out_loss=loss)
runner = tx.ModelRunner(model)
runner.config_optimizer(optimizer=tf.train.AdamOptimizer(learning_rate=0.001))
runner.init_vars()
for data_batch in data:
data_batch = np.array(data_batch)
ctx_vector = data_batch[:, :-1]
label = data_batch[:, -1:]
loss = runner.train(ctx_vector, label, output_loss=True)
print(np.mean(loss))
"""
# prediction = tf.argmax(h0, 1)
embed_dim,
shape=[None, seq_size, embed_dim],
wrap_fn=lambda tensor: tf.reshape(tensor, [-1, seq_size, embed_dim]))
out = tx.WrapLayer(out, embed_dim, wrap_fn=lambda tensor: tensor[0])
# apply rnn cell to single input batch
with tf.name_scope("rnn"):
rnn1 = RNNCell(out, 4, name="rnn1")
rnn2 = rnn1.reuse_with(out, state=rnn1, name="rnn2")
rnn3 = rnn1.reuse_with(out, state=rnn2, name="rnn3")
# setup optimizer
optimizer = tx.AMSGrad(learning_rate=0.01)
model = tx.Model(run_inputs=in_layer, run_outputs=[rnn1, rnn2, rnn3])
runner = tx.ModelRunner(model)
runner.set_session(runtime_stats=True)
runner.log_graph(logdir="/tmp")
print("graph written")
runner.init_vars()
# need to fix the runner interface to allow for lists to be received
data = np.array([[0, 1], [1, 0]])
targets = np.array([[2], [3]])
result = runner.run(data)
for i, r in enumerate(result):
sampled_loss = tf.reduce_mean(sampled_loss +
tf.nn.l2_loss(lookup.weights) * 1e-6)
tqdm.write("loss: {}".format(sampled_loss.eval()))
lr = tensorx.layers.Param(0.0005)
# opt = tf.train.RMSPropOptimizer(learning_rate=lr.tensor)
opt = tx.AMSGrad(learning_rate=lr.tensor)
# opt = tf.train.GradientDescentOptimizer(learning_rate=lr.tensor)
# = opt.minimize(sampled_loss)
# sess.run(tf.variables_initializer(opt.variables()))
model = tx.Model(run_inputs=input_layer,
train_in_loss=input_labels,
train_out_loss=sampled_loss,
eval_out_score=val_loss)
runner = tx.ModelRunner(model)
runner.set_session()
runner.config_optimizer(opt,
optimizer_params=lr,
gradient_op=lambda grad: tf.clip_by_norm(grad, 1.0))
avg_nce = []
avg_ppl = []
avg_loss = []
current_nce = []
current_loss = []
lr.value = 0.1
# last = tf.gather(val, int(val.get_shape()[0]) - 1)
last = val[-1]
lstm_out = tx.TensorLayer(last, n_hidden)
logits = tx.Linear(lstm_out, vocab_size, bias=True)
out = tx.Activation(logits, tx.softmax)
labels = tx.dense_one_hot(loss_inputs.tensor, vocab_size)
loss = tf.reduce_mean(tx.categorical_cross_entropy(labels=labels, logits=logits.tensor))
# setup optimizer
optimizer = tx.AMSGrad(learning_rate=0.01)
model = tx.Model(run_inputs=in_layer, run_outputs=out,
train_inputs=in_layer, train_outputs=out,
train_in_loss=loss_inputs, train_out_loss=loss,
eval_out_score=loss, eval_in_score=loss_inputs)
print(model.feedable_train())
runner = tx.ModelRunner(model)
runner.config_optimizer(optimizer)
runner.init_vars()
# need to fix the runner interface to allow for lists to be received
data = np.array([[0, 1], [1, 0]])
targets = np.array([[2], [3]])
for i in tqdm(range(10000)):
runner.train(model_input_data=data, loss_input_data=targets)
