def test_model_run(): data1 = tf.constant([[1., 1.]]) x = tx.Input(n_units=2, name="x", constant=False) labels = tx.Input(n_units=2, name="y_", constant=False) y = tx.Linear(x, 2, name="y") out1 = tx.Activation(y, tf.nn.softmax) out2 = tx.Activation(y, tf.nn.softmax) @tx.layer(n_units=2, name="loss") def loss(pred, labs): return tf.losses.categorical_crossentropy(labs, pred) model = tx.Model(run_inputs=x, run_outputs=[out1, out2], train_inputs=[x, labels], train_outputs=out1, train_loss=loss(out1, labels)) model.set_optimizer(tf.optimizers.SGD, lr=0.5) result1 = model.run({x: data1}) result2 = model.run([data1]) assert tx.tensor_equal(result1[0], result2[0]) assert tx.tensor_equal(result1[1], result2[1]) result3 = model.run({x: data1}, compiled_graph=True) assert tx.tensor_equal(result3[0], result2[0]) assert tx.tensor_equal(result3[1], result2[1])
def test_loss_model_dependencies(): inputs = tx.Input(n_units=2, name="x", constant=False) labels = tx.Input(n_units=2, name="y_", constant=False) y = tx.Linear(inputs, 2, name="y") out1 = tx.Activation(y, tf.nn.softmax, name="out1") out2 = tx.Activation(y, tf.nn.softmax, name="out2") @tx.layer(n_units=2, name="loss") def loss(pred, labs): return tf.losses.categorical_crossentropy(labs, pred) logging.basicConfig(level=logging.DEBUG) model = tx.Model(run_inputs=inputs, run_outputs=[out1, out2], train_inputs=[inputs, labels], train_outputs=[out2, out1], train_loss=loss(out1, labels)) lr = tx.Param(0.5) opt = model.set_optimizer(tf.optimizers.SGD, lr=lr) assert isinstance(opt, tf.optimizers.Optimizer) it = model.train_graph.dependency_iter() layers = list(it) assert layers[0] is inputs assert layers[1] is labels assert len(layers) == 6
def test_add_optimizer(): target = tx.Constant([[1.]]) inputs = tx.Input(n_units=2, name="inputs") output = tx.Linear(inputs, n_units=1, name="y") loss = tx.Lambda(target, output, fn=tf.nn.softmax_cross_entropy_with_logits, name="xent") m = tx.Model(run_outputs=output, train_inputs=[inputs, target], train_loss=loss) lr = tx.Param(init_value=0.2, name="lr") optimizer1 = m.set_optimizer(tf.optimizers.SGD, lr=lr) optimizer2: tf.optimizers.Optimizer = m.optimizer assert optimizer1 == optimizer2 # optimizer is hashable opt_dict = {optimizer1: 0, optimizer2: 1} assert optimizer1 in opt_dict assert opt_dict[optimizer1] == 1 lr.value = 0.3 assert np.float32(0.3) == optimizer1.lr.numpy()
def test_set_optimizer(): x = tx.Input(n_units=2, name="x", constant=False) labels = tx.Input(n_units=2, name="labels", constant=False) y = tx.Linear(x, 2, name="y") out1 = tx.Activation(y, tf.nn.softmax) out2 = tx.Activation(y, tf.nn.softmax) @tx.layer(n_units=2, name="loss") def loss(pred, labs): return tf.losses.categorical_crossentropy(labs, pred) model = tx.Model(run_inputs=x, run_outputs=[out1, out2], train_inputs=[x, labels], train_outputs=[out2, out1], train_loss=loss(out1, labels)) lr = tx.Param(0.5) opt = model.set_optimizer(tf.optimizers.SGD, learning_rate=lr, clipnorm=0.1) assert isinstance(opt, tf.optimizers.Optimizer) assert model.optimizer.get_config()["learning_rate"] == 0.5 data1 = [[1., 1.], [1., 1.]] data2 = tf.constant([[0., 1.], [0., 1.]]) params = model.optimizer_params[model.optimizer] data_dict, params_dict = tx.Model.parse_input( { x: data1, "learning_rate": 0.2 }, model.run_graph.in_nodes, params) assert len(data_dict) == 1 assert len(params_dict) == 1 assert model.optimizer_params[opt]["learning_rate"] is lr result1 = model.train_step({x: data1, labels: data2}) result2 = model.train_step([data1, data2]) assert len(result1) == 3 assert len(result2) == 3 assert tf.reduce_all(tf.less(result2[-1], result1[-1])) result1 = model.run({x: np.array(data1, dtype=np.float32)}) result2 = model.run([data1]) result3 = model.run(np.array(data1, np.float32)) x.value = data1 o2 = out2() o1 = out1() result4 = (o2, o1) for i in range(2): assert tx.tensor_equal(result1[i], result2[i]) assert tx.tensor_equal(result1[i], result3[i]) assert tx.tensor_equal(result1[i], result4[i])
def test_model_vars(): target = tx.Constant([[1.]]) inputs = tx.Input(n_units=2, name="inputs", constant=False) output = tx.Linear(inputs, n_units=1, name="y") loss = tx.Lambda(target, output, fn=tf.nn.softmax_cross_entropy_with_logits, name="xent") m = tx.Model(run_outputs=output, train_inputs=[inputs, target], train_loss=loss) assert m.trainable_variables == output.trainable_variables
def test_model_var_inputs(): # wanted to test when our train graph has more inputs that do not need to be fed (e.g. variable state) n_features = 5 embed_size = 4 hidden_dim = 3 seq_size = 3 out_size = 2 batch_size = 2 x = tx.Input(np.random.random([batch_size, seq_size]), n_units=seq_size, dtype=tf.int32) y = tx.Input(np.random.random([batch_size, out_size]), n_units=out_size, dtype=tf.float32) lookup = tx.Lookup(x, seq_size=seq_size, embedding_shape=[n_features, embed_size]) # seq = lookup.permute_batch_time() seq = tx.Transpose(lookup, [1, 0, 2]) rnn1 = tx.RNN(seq, cell_config=tx.RNNCell.config(n_units=hidden_dim)) y_ = tx.Linear(rnn1[seq_size - 1], n_units=out_size) # y_ = tx.Linear(tx.SeqConcat(lookup, seq_size=seq_size), n_units=out_size) # @tx.layer(n_units=2, dtype=tf.float32, name="loss") # def loss(pred, labels): # return tx.mse(pred, labels) model = tx.Model(run_inputs=x, run_outputs=y_, train_inputs=[x, y], train_outputs=y_, train_loss=tx.MSE(y_, y)) # model.draw("test.pdf") model.set_optimizer(tf.optimizers.SGD, lr=0.5) data1 = [[0, 1, 2], [2, 1, 0]] data2 = [[0., 1.], [1., 0.]] model.train_step(input_feed={x: data1, y: data2})
def test_model_train(): x = tx.Input(n_units=2, name="x", constant=False) labels = tx.Input(n_units=2, name="labels", constant=False) y = tx.Linear(x, 2, name="y1", add_bias=False) out1 = tx.Activation(y, tf.nn.softmax) out2 = tx.Activation(y, tf.nn.softmax) @tx.layer(n_units=2, name="loss") def loss(pred, labs): return tf.losses.categorical_crossentropy(labs, pred) model = tx.Model(run_inputs=x, run_outputs=[out1, out2], train_inputs=[x, labels], train_outputs=[out2, out1], train_loss=loss(out1, labels)) lr = tx.Param(0.5) opt = model.set_optimizer(tf.optimizers.SGD, learning_rate=lr, clipnorm=0.1) data1 = [[1., 1.], [1., 1.]] data2 = [[0., 1.], [0., 1.]] w1 = y.weights.numpy() epochs = 100 model.train(train_data=[{x: data1, labels: data2}], epochs=epochs) w2 = y.weights.value() y.weights.assign(w1) for _ in range(epochs): model.train_step(input_feed={x: data1, labels: data2}) w3 = y.weights.value() assert tx.tensor_equal(w2, w3)
data = batch_it(data, batch_size) label_layer = tx.Input(1) in_layer = tx.Input(M) f1 = tx.FC(in_layer, 512, activation=tf.nn.tanh) f2 = tx.FC(f1, 512, activation=tf.nn.relu) fm = tx.Highway(f1, f2, carry_gate=True) out = tx.Linear(f2, 1) out_prob = tx.Activation(out, fn=tx.sigmoid) loss = tx.binary_cross_entropy(labels=label_layer.tensor, logits=out.tensor) model = tx.Model(run_inputs=in_layer, run_outputs=out_prob, train_in_loss=label_layer, train_out_loss=loss) runner = tx.ModelRunner(model) runner.config_optimizer(optimizer=tf.train.AdamOptimizer(learning_rate=0.001)) runner.init_vars() for data_batch in data: data_batch = np.array(data_batch) ctx_vector = data_batch[:, :-1] label = data_batch[:, -1:] loss = runner.train(ctx_vector, label, output_loss=True) print(np.mean(loss)) """ # prediction = tf.argmax(h0, 1)
embed_dim, shape=[None, seq_size, embed_dim], wrap_fn=lambda tensor: tf.reshape(tensor, [-1, seq_size, embed_dim])) out = tx.WrapLayer(out, embed_dim, wrap_fn=lambda tensor: tensor[0]) # apply rnn cell to single input batch with tf.name_scope("rnn"): rnn1 = RNNCell(out, 4, name="rnn1") rnn2 = rnn1.reuse_with(out, state=rnn1, name="rnn2") rnn3 = rnn1.reuse_with(out, state=rnn2, name="rnn3") # setup optimizer optimizer = tx.AMSGrad(learning_rate=0.01) model = tx.Model(run_inputs=in_layer, run_outputs=[rnn1, rnn2, rnn3]) runner = tx.ModelRunner(model) runner.set_session(runtime_stats=True) runner.log_graph(logdir="/tmp") print("graph written") runner.init_vars() # need to fix the runner interface to allow for lists to be received data = np.array([[0, 1], [1, 0]]) targets = np.array([[2], [3]]) result = runner.run(data) for i, r in enumerate(result):
sampled_loss = tf.reduce_mean(sampled_loss + tf.nn.l2_loss(lookup.weights) * 1e-6) tqdm.write("loss: {}".format(sampled_loss.eval())) lr = tensorx.layers.Param(0.0005) # opt = tf.train.RMSPropOptimizer(learning_rate=lr.tensor) opt = tx.AMSGrad(learning_rate=lr.tensor) # opt = tf.train.GradientDescentOptimizer(learning_rate=lr.tensor) # = opt.minimize(sampled_loss) # sess.run(tf.variables_initializer(opt.variables())) model = tx.Model(run_inputs=input_layer, train_in_loss=input_labels, train_out_loss=sampled_loss, eval_out_score=val_loss) runner = tx.ModelRunner(model) runner.set_session() runner.config_optimizer(opt, optimizer_params=lr, gradient_op=lambda grad: tf.clip_by_norm(grad, 1.0)) avg_nce = [] avg_ppl = [] avg_loss = [] current_nce = [] current_loss = [] lr.value = 0.1
# last = tf.gather(val, int(val.get_shape()[0]) - 1) last = val[-1] lstm_out = tx.TensorLayer(last, n_hidden) logits = tx.Linear(lstm_out, vocab_size, bias=True) out = tx.Activation(logits, tx.softmax) labels = tx.dense_one_hot(loss_inputs.tensor, vocab_size) loss = tf.reduce_mean(tx.categorical_cross_entropy(labels=labels, logits=logits.tensor)) # setup optimizer optimizer = tx.AMSGrad(learning_rate=0.01) model = tx.Model(run_inputs=in_layer, run_outputs=out, train_inputs=in_layer, train_outputs=out, train_in_loss=loss_inputs, train_out_loss=loss, eval_out_score=loss, eval_in_score=loss_inputs) print(model.feedable_train()) runner = tx.ModelRunner(model) runner.config_optimizer(optimizer) runner.init_vars() # need to fix the runner interface to allow for lists to be received data = np.array([[0, 1], [1, 0]]) targets = np.array([[2], [3]]) for i in tqdm(range(10000)): runner.train(model_input_data=data, loss_input_data=targets)