def train(conf_dict):
"""
train
"""
training_mode = conf_dict["training_mode"]
net = utility.import_object(
conf_dict["net_py"], conf_dict["net_class"])(conf_dict)
if training_mode == "pointwise":
datafeed = datafeeds.TFPointwisePaddingData(conf_dict)
input_l, input_r, label_y = datafeed.ops()
pred = net.predict(input_l, input_r)
output_prob = tf.nn.softmax(pred, -1, name="output_prob")
loss_layer = utility.import_object(
conf_dict["loss_py"], conf_dict["loss_class"])()
loss = loss_layer.ops(pred, label_y)
elif training_mode == "pairwise":
datafeed = datafeeds.TFPairwisePaddingData(conf_dict)
input_l, input_r, neg_input = datafeed.ops()
pos_score = net.predict(input_l, input_r)
output_prob = tf.identity(pos_score, name="output_preb")
neg_score = net.predict(input_l, neg_input)
loss_layer = utility.import_object(
conf_dict["loss_py"], conf_dict["loss_class"])(conf_dict)
loss = loss_layer.ops(pos_score, neg_score)
else:
print(sys.stderr, "training mode not supported")
sys.exit(1)
# define optimizer
lr = float(conf_dict["learning_rate"])
optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss)
# run_trainer
controler.run_trainer(loss, optimizer, conf_dict)
def train(conf_dict):
tf.compat.v1.reset_default_graph()
net = utility.import_object(conf_dict["net_py"],
conf_dict["net_class"])(conf_dict)
datafeed = datafeeds.TFPointwisePaddingData(conf_dict)
input_l, input_r, label_y = datafeed.ops()
pred = net.predict(input_l, input_r)
loss_layer = utility.import_object(conf_dict["loss_py"],
conf_dict["loss_class"])()
loss = loss_layer.ops(pred, label_y)
# define optimizer
lr = float(conf_dict["learning_rate"])
optimizer = tf.compat.v1.train.AdamOptimizer(
learning_rate=lr).minimize(loss)
# run_trainer
controler.run_trainer(loss, optimizer, conf_dict)
def freeze(conf_dict):
"""
freeze net for c api predict
"""
model_path = conf_dict["save_path"]
freeze_path = conf_dict["freeze_path"]
training_mode = conf_dict["training_mode"]
graph = tf.Graph()
with graph.as_default():
net = utility.import_object(conf_dict["net_py"],
conf_dict["net_class"])(conf_dict)
test_l = dict([(u, tf.placeholder(tf.int32, [None, v], name=u))
for (u, v) in dict(conf_dict["left_slots"]).iteritems()
])
test_r = dict([(u, tf.placeholder(tf.int32, [None, v], name=u))
for (u,
v) in dict(conf_dict["right_slots"]).iteritems()])
pred = net.predict(test_l, test_r)
if training_mode == "pointwise":
output_prob = tf.nn.softmax(pred, -1, name="output_prob")
elif training_mode == "pairwise":
output_prob = tf.identity(pred, name="output_prob")
restore_saver = tf.train.Saver()
with tf.Session(graph=graph) as sess:
sess.run(tf.global_variables_initializer())
restore_saver.restore(sess, model_path)
output_graph_def = tf.graph_util.\
convert_variables_to_constants(sess, sess.graph_def, ["output_prob"])
tf.train.write_graph(output_graph_def, '.', freeze_path, as_text=False)
def predict(conf_dict):
"""
predict
"""
net = utility.import_object(conf_dict["net_py"],
conf_dict["net_class"])(conf_dict)
conf_dict.update({
"num_epochs": "1",
"batch_size": "800",
"shuffle": "0",
"train_file": conf_dict["test_file"]
})
test_datafeed = datafeeds.TFPointwisePaddingData(conf_dict)
test_l, test_r, test_y = test_datafeed.ops()
# test network
pred = net.predict(test_l, test_r)
loss_layer = utility.import_object(conf_dict["loss_py"],
conf_dict["loss_class"])(conf_dict)
loss = loss_layer.ops(pred, test_y)
controler.run_predict(pred, test_y, conf_dict, loss)
def freeze(conf_dict):
"""
freeze net for c api predict
"""
net = utility.import_object(conf_dict["net_py"],
conf_dict["net_class"])(conf_dict)
test_l = dict([(u, tf.placeholder(tf.int32, [None, v], name=u))
for (u, v) in dict(conf_dict["left_slots"]).iteritems()])
test_r = dict([(u, tf.placeholder(tf.int32, [None, v], name=u))
for (u, v) in dict(conf_dict["right_slots"]).iteritems()])
pred = net.predict(test_l, test_r)
controler.graph_save(pred, conf_dict)
def predict(conf_dict):
tf.compat.v1.reset_default_graph()
net = utility.import_object(conf_dict["net_py"],
conf_dict["net_class"])(conf_dict)
conf_dict.update({
"batch_size": "1",
"shuffle": "0",
"train_file": conf_dict["test_file"]
})
if "dropout_rate" in conf_dict:
conf_dict.update({"dropout_rate": 1.0})
test_datafeed = datafeeds.TFPointwisePaddingData(conf_dict)
test_l, test_r, test_y = test_datafeed.ops()
pred = net.predict(test_l, test_r)
controler.run_predict(pred, test_y, conf_dict)
def predict(conf_dict):
"""
predict
"""
#导入网络类
net = utility.import_object(
conf_dict["net_py"], conf_dict["net_class"])(conf_dict)
conf_dict.update({"num_epochs": "1", "batch_size": "1",
"shuffle": "0", "train_file": conf_dict["test_file"]})
test_datafeed = datafeeds.TFPointwisePaddingData(conf_dict)
#建立TF_Record的数据读取通道
test_l, test_r, test_y = test_datafeed.ops()
# test network
pred = net.predict(test_l, test_r)
controler.run_predict(pred, test_y, conf_dict)
