def create_model(params, is_train):
"""Creates transformer model."""
with tf.name_scope("model"):
if is_train:
inputs = tf.keras.layers.Input((None,), dtype="int64", name="inputs")
targets = tf.keras.layers.Input((None,), dtype="int64", name="targets")
px = tf.keras.layers.Input((3,), dtype="float32", name="px")
py = tf.keras.layers.Input((3,), dtype="float32", name="py")
# sim = tf.keras.layers.Input((1,), dtype="float32", name="sim")
internal_model = Transformer(params, name="transformer_v2")
logits = internal_model([inputs, px, py, targets], training=is_train)
# logits = internal_model([inputs, px, sim, py, targets], training=is_train)
vocab_size = params["vocab_size"]
label_smoothing = params["label_smoothing"]
if params["enable_metrics_in_training"]:
logits = metrics.MetricLayer(vocab_size)([logits, targets])
logits = tf.keras.layers.Lambda(lambda x: x, name="logits")(logits)
model = tf.keras.Model([inputs, px, py, targets], logits)
# model = tf.keras.Model([inputs, px, sim, py, targets], logits)
loss = metrics.transformer_loss(
logits, targets, label_smoothing, vocab_size)
model.add_loss(loss)
return model
else:
inputs = tf.keras.layers.Input((None,), dtype="int64", name="inputs")
px = tf.keras.layers.Input((3,), dtype="float32", name="px")
py = tf.keras.layers.Input((3,), dtype="float32", name="py")
# sim = tf.keras.layers.Input((1,), dtype="float32", name="sim")
internal_model = Transformer(params, name="transformer_v2")
# ret = internal_model([inputs, px, sim, py], training=is_train)
ret = internal_model([inputs, px, py], training=is_train)
outputs, scores = ret["outputs"], ret["scores"]
return tf.keras.Model([inputs, px, py], [outputs, scores])
def create_model(params, is_train):
"""Creates transformer model."""
with tf.name_scope("model"):
if is_train:
inputs = tf.keras.layers.Input((None,), dtype="int64", name="inputs")
targets = tf.keras.layers.Input((None,), dtype="int64", name="targets")
px = tf.keras.layers.Input((3,), dtype="float32", name="px")
py = tf.keras.layers.Input((3,), dtype="float32", name="py")
# sim = tf.keras.layers.Input((1,), dtype="float32", name="sim")
internal_model = Transformer(params, name="optgen_v11")
logits = internal_model([inputs, px, py, targets], training=is_train)
# logits = internal_model([inputs, px, sim, py, targets], training=is_train)
vocab_size = params["vocab_size"]
label_smoothing = params["label_smoothing"]
if params["enable_metrics_in_training"]:
logits = metrics.MetricLayer(vocab_size)([logits, targets])
logits = tf.keras.layers.Lambda(lambda x: x, name="logits")(logits)
valnet_hat = get_valnet(params, logits)
propnet_hat = get_propnet(params, logits)
model = tf.keras.Model([inputs, px, py, targets], logits)
# model = tf.keras.Model([inputs, px, sim, py, targets], logits)
loss = metrics.transformer_loss(
logits, targets, label_smoothing, vocab_size)
model.add_loss(loss)
valnet_true = tf.ones_like(valnet_hat)
loss_valnet = tf.keras.losses.binary_crossentropy(
valnet_true,
valnet_hat,
from_logits=False,
label_smoothing=0
)
model.add_loss(tf.reduce_sum(loss_valnet))
propnet_true = tf.keras.layers.Lambda(lambda x: x)(py)
loss_propnet = tf.keras.losses.mse(propnet_true, propnet_hat)
model.add_loss(tf.reduce_sum(loss_propnet))
return model
else:
inputs = tf.keras.layers.Input((None,), dtype="int64", name="inputs")
px = tf.keras.layers.Input((3,), dtype="float32", name="px")
py = tf.keras.layers.Input((3,), dtype="float32", name="py")
# sim = tf.keras.layers.Input((1,), dtype="float32", name="sim")
internal_model = Transformer(params, name="transformer_v2")
# ret = internal_model([inputs, px, sim, py], training=is_train)
ret = internal_model([inputs, px, py], training=is_train)
outputs, scores = ret["outputs"], ret["scores"]
return tf.keras.Model([inputs, px, py], [outputs, scores])
def create_model(params, is_train):
"""Creates transformer model."""
# aux_layers = None
aux_layers = get_aux_layers(params)
valnet_embedding_softmax_layer, valnet_bi_lstm_layer, valnet_output_layer, \
propnet_embedding_softmax_layer, propnet_bi_lstm_layer, propnet_idense_layer, propnet_output_layer, \
simnet_embedding_softmax_layer, simnet_bi_lstm_layer, simnet_idense_layer, simnet_output_layer = aux_layers
with tf.name_scope("model"):
if is_train:
inputs = tf.keras.layers.Input((None, ),
dtype="int64",
name="inputs")
targets = tf.keras.layers.Input((None, ),
dtype="int64",
name="targets")
px = tf.keras.layers.Input((3, ), dtype="float32", name="px")
py = tf.keras.layers.Input((3, ), dtype="float32", name="py")
# sim = tf.keras.layers.Input((1,), dtype="float32", name="sim")
internal_model = Transformer(params, aux_layers, name="optgen_v21")
logits = internal_model([inputs, px, py, targets],
training=is_train)
# logits = internal_model([inputs, px, sim, py, targets], training=is_train)
vocab_size = params["vocab_size"]
label_smoothing = params["label_smoothing"]
if params["enable_metrics_in_training"]:
logits = metrics.MetricLayer(vocab_size)([logits, targets])
logits = tf.keras.layers.Lambda(lambda x: x, name="logits")(logits)
model = tf.keras.Model([inputs, px, py, targets], logits)
# model = tf.keras.Model([inputs, px, sim, py, targets], logits)
loss = metrics.transformer_loss(logits, targets, label_smoothing,
vocab_size)
model.add_loss(loss)
if params["use_valnet"] == 1:
valnet_hat = get_valnet(logits, valnet_embedding_softmax_layer,
valnet_bi_lstm_layer,
valnet_output_layer)
valnet_true = tf.ones_like(valnet_hat)
loss_valnet = tf.keras.losses.binary_crossentropy(
valnet_true,
valnet_hat,
from_logits=False,
label_smoothing=0)
model.add_loss(tf.reduce_sum(loss_valnet))
if params["use_propnet"] == 1:
propnet_hat = get_propnet(logits,
propnet_embedding_softmax_layer,
propnet_bi_lstm_layer,
propnet_idense_layer,
propnet_output_layer)
propnet_true = tf.keras.layers.Lambda(lambda x: x)(py)
loss_propnet = tf.keras.losses.mse(propnet_true, propnet_hat)
model.add_loss(tf.reduce_sum(loss_propnet))
if params["use_simnet"] == 1:
simnet_hat = get_simnet(logits, inputs,
simnet_embedding_softmax_layer,
simnet_bi_lstm_layer,
simnet_idense_layer,
simnet_output_layer)
simnet_true = tf.ones_like(simnet_hat)
loss_simnet = tf.keras.losses.binary_crossentropy(
simnet_true,
simnet_hat,
from_logits=False,
label_smoothing=0)
model.add_loss(tf.reduce_sum(loss_simnet))
return model
else:
inputs = tf.keras.layers.Input((None, ),
dtype="int64",
name="inputs")
px = tf.keras.layers.Input((3, ), dtype="float32", name="px")
py = tf.keras.layers.Input((3, ), dtype="float32", name="py")
sim = tf.keras.layers.Input((1, ), dtype="float32", name="sim")
internal_model = Transformer(params, aux_layers, name="optgen_v21")
# ret = internal_model([inputs, px, sim, py], training=is_train)
ret = internal_model([inputs, px, py], training=is_train)
# tf.print("retscores", ret["scores"].shape, output_stream=sys.stderr)
# tf.print("bbb", bbb.shape, output_stream=sys.stderr)
# tf.print("bbb", bbb.shape, output_stream=sys.stderr)
scores = ret["scores"]
if params["use_valnet"] == 1:
aaa = get_valnet_with_id(inputs,
valnet_embedding_softmax_layer,
valnet_bi_lstm_layer,
valnet_output_layer)
aaa = tf.keras.layers.Lambda(
lambda x: tf.reduce_sum(x, axis=-1), name="dummy1")(aaa)
scores += aaa
if params["use_propnet"] == 1:
bbb = get_propnet_with_id(inputs,
propnet_embedding_softmax_layer,
propnet_bi_lstm_layer,
propnet_idense_layer,
propnet_output_layer)
bbb = tf.keras.layers.Lambda(
lambda x: tf.reduce_sum(x, axis=-1), name="dummy2")(bbb)
scores += bbb
if params["use_simnet"] == 1:
ccc = get_simnet_with_id(inputs, inputs,
simnet_embedding_softmax_layer,
simnet_bi_lstm_layer,
simnet_idense_layer,
simnet_output_layer)
ccc = tf.keras.layers.Lambda(
lambda x: tf.reduce_sum(x, axis=-1), name="dummy3")(ccc)
scores += ccc
outputs = ret["outputs"]
# outputs, scores = ret["outputs"], ret["scores"]
return tf.keras.Model([inputs, px, py], [outputs, scores])
def create_model(params, is_train):
"""Creates transformer model."""
with tf.name_scope("model"):
if is_train:
inputs = tf.keras.layers.Input((None, ),
dtype="int64",
name="inputs")
targets = tf.keras.layers.Input((None, ),
dtype="int64",
name="targets")
px = tf.keras.layers.Input((3, ), dtype="float32", name="px")
py = tf.keras.layers.Input((3, ), dtype="float32", name="py")
# sim = tf.keras.layers.Input((1,), dtype="float32", name="sim")
internal_model = Transformer(params, name="optgen_v13")
logits = internal_model([inputs, px, py, targets],
training=is_train)
# logits = internal_model([inputs, px, sim, py, targets], training=is_train)
vocab_size = params["vocab_size"]
label_smoothing = params["label_smoothing"]
if params["enable_metrics_in_training"]:
logits = metrics.MetricLayer(vocab_size)([logits, targets])
logits = tf.keras.layers.Lambda(lambda x: x, name="logits")(logits)
valnet_embedding_softmax_layer = embedding_layer.EmbeddingFreezable(
params["vocab_size"],
params["valnet_hidden_size"],
trainable=False)
valnet_bi_lstm_layer = tf.keras.layers.Bidirectional(
tf.keras.layers.LSTM(params["valnet_hidden_size"],
name="valnet_lstm",
trainable=False))
valnet_output_layer = tf.keras.layers.Dense(
1,
use_bias=True,
activation=tf.nn.sigmoid,
name="valnet_output",
trainable=False)
valnet_hat = get_valnet(logits, valnet_embedding_softmax_layer,
valnet_bi_lstm_layer, valnet_output_layer)
propnet_embedding_softmax_layer = embedding_layer.EmbeddingFreezable(
params["vocab_size"],
params["propnet_hidden_size"],
trainable=False)
propnet_bi_lstm_layer = tf.keras.layers.Bidirectional(
tf.keras.layers.LSTM(params["propnet_hidden_size"],
name="prop_lstm",
trainable=False))
propnet_idense_layer = tf.keras.layers.Dense(
100,
use_bias=True,
activation=tf.nn.sigmoid,
name="propnet_idense",
trainable=False)
propnet_output_layer = tf.keras.layers.Dense(
3,
use_bias=True,
activation=tf.nn.sigmoid,
name="propnet_output",
trainable=False)
propnet_hat = get_propnet(logits, propnet_embedding_softmax_layer,
propnet_bi_lstm_layer,
propnet_idense_layer,
propnet_output_layer)
simnet_embedding_softmax_layer = embedding_layer.EmbeddingFreezable(
params["vocab_size"],
params["simnet_hidden_size"],
trainable=False)
simnet_bi_lstm_layer = tf.keras.layers.Bidirectional(
tf.keras.layers.LSTM(params["simnet_hidden_size"],
name="simnet_lstm",
trainable=False))
simnet_idense_layer = tf.keras.layers.Dense(100,
use_bias=True,
activation=tf.nn.relu,
name="simnet_idense",
trainable=False)
simnet_output_layer = tf.keras.layers.Dense(
1,
use_bias=True,
activation=tf.nn.sigmoid,
name="simnet_output",
trainable=False)
simnet_hat = get_simnet(logits, inputs,
simnet_embedding_softmax_layer,
simnet_bi_lstm_layer, simnet_idense_layer,
simnet_output_layer)
model = tf.keras.Model([inputs, px, py, targets], logits)
# model = tf.keras.Model([inputs, px, sim, py, targets], logits)
loss = metrics.transformer_loss(logits, targets, label_smoothing,
vocab_size)
model.add_loss(loss)
valnet_true = tf.ones_like(valnet_hat)
loss_valnet = tf.keras.losses.binary_crossentropy(
valnet_true, valnet_hat, from_logits=False, label_smoothing=0)
model.add_loss(tf.reduce_sum(loss_valnet))
propnet_true = tf.keras.layers.Lambda(lambda x: x)(py)
loss_propnet = tf.keras.losses.mse(propnet_true, propnet_hat)
model.add_loss(tf.reduce_sum(loss_propnet))
simnet_true = tf.ones_like(simnet_hat)
loss_simnet = tf.keras.losses.binary_crossentropy(
simnet_true, simnet_hat, from_logits=False, label_smoothing=0)
model.add_loss(tf.reduce_sum(loss_simnet))
return model
else:
inputs = tf.keras.layers.Input((None, ),
dtype="int64",
name="inputs")
px = tf.keras.layers.Input((3, ), dtype="float32", name="px")
py = tf.keras.layers.Input((3, ), dtype="float32", name="py")
# sim = tf.keras.layers.Input((1,), dtype="float32", name="sim")
internal_model = Transformer(params, name="optgen_v13")
# ret = internal_model([inputs, px, sim, py], training=is_train)
ret = internal_model([inputs, px, py], training=is_train)
outputs, scores = ret["outputs"], ret["scores"]
return tf.keras.Model([inputs, px, py], [outputs, scores])