def main():
if len(sys.argv) == 1:
print("Usage: train.py --job_name --task_index --deviceid\n"
" --job_name: One of 'ps', 'worker'\n"
" --task_index: Index of task within the job\n"
" --deviceid: GPU ID: 0-7\n")
exit(1)
# with global variable 'log' and 'args'
parseArgs()
args.log_prefix = args.job_name + ':' + str(args.task_index) + ':' + str(
args.deviceid) + '_'
initenviroment()
server, cluster = initDistributed()
if args.isps is True:
args.message("Parameter server joined!")
args.write_args(args)
server.join()
else:
model = graph_moudle.Model()
## Step 1: Init tensorflow:
sess = initworker(server, cluster, model)
## Step 3: Init dataset
dataset = initDataset(model.data_proc)
args.write_args(args)
if args.ischief:
args.file_copy(['dev'])
## Step 4: Run it
trainLoop(model, sess, dataset)
def main():
parseArgs(args)
args.log_dir_path = args.output + os.path.sep + 'test_' + os.path.split(
args.checkpoint)[1]
args.log_prefix = args.a0_model_name + '_'
if not os.path.exists(args.log_dir_path):
os.makedirs(args.log_dir_path)
configproto = tf.ConfigProto()
configproto.gpu_options.allow_growth = True
configproto.allow_soft_placement = True
args.batchsize = 400
test_batchsize = args.batchsize
with tf.Graph().as_default(), tf.Session(config=configproto) as sess:
model = graph_moudle.Model()
model.init_global_step()
vt, vs, vo = model.model_setup()
tf.initialize_all_variables().run()
args.message('Loading trained model ' + str(args.checkpoint))
model.saver.restore(sess, args.checkpoint)
args.write_variables(vt)
args.write_args(args)
print('Predicting ' + str(args.fake_input))
fake_input_file = open(args.fake_input, 'rb')
merge_data_file = open(args.merge_data, 'rb')
new_feature_file = open(os.path.split(args.merge_data)[0] + os.path.sep +\
os.path.split(args.checkpoint)[1] + '.ltr', 'wb')
pairs = fake_input_file.readlines()
features = merge_data_file.readlines()
n_test = len(pairs)
n_batches = int(n_test / test_batchsize)
if n_test % test_batchsize != 0:
n_batches += 1
for i in range(n_batches):
data_proc_slice = model.data_proc(
pairs[i * test_batchsize:(i + 1) * test_batchsize],
n_samples=1)
score = model.run_score(sess, data_proc_slice)[0]
for j in range(0, len(score)):
line_num = i * test_batchsize + j
if line_num >= n_test:
break
feature_line = features[line_num]
feature_text = feature_line.split('#')
feature_text[0] += '6004:' + str(score[j][0]) + ' '
new_line = '#'.join(feature_text)
new_line = ''.join(new_line.split(',')) # special judgement
new_feature_file.write(new_line)
out_str = "Print log of " + str(
args.fake_input
) + " with checkpoint " + args.checkpoint + " finished!"
args.message(out_str)
fake_input_file.close()
merge_data_file.close()
new_feature_file.close()
def main():
parseArgs(args)
configproto = tf.ConfigProto()
configproto.gpu_options.allow_growth = True
configproto.allow_soft_placement = True
ops.reset_default_graph()
with tf.Graph().as_default(), tf.Session(config=configproto) as sess:
model = graph_moudle.Model()
model.init_global_step()
vt, vs, vo = model.model_setup()
tf.global_variables_initializer().run()
args.message('Loading trained model ' + str(args.checkpoint))
model.saver.restore(sess, args.checkpoint)
for var in tf.trainable_varisbles():
print var.name
args.write_variables(vt)
args.write_args(args)
def main():
checkName = "/search/odin/data/wangyuan/pycharmProjects/siameseQrnn/model/20180131/siamQrnnCkpt_1"
configproto = tf.ConfigProto()
configproto.gpu_options.allow_growth = True
configproto.allow_soft_placement = True
ops.reset_default_graph()
with tf.Graph().as_default(), tf.Session(config=configproto) as sess:
model = graph_moudle.Model()
model.init_global_step()
vt, vs, vo = model.model_setup()
tf.global_variables_initializer().run()
args.message('Loading trained model ' + checkName)
model.saver.restore(sess, checkName)
variable_names = tf.trainable_variables()
values = sess.run(variable_names)
for k, v in zip(variable_names, values):
print("Variable: ", k)
print("Shape: ", v.shape)
print(v)
def main():
configproto = tf.ConfigProto()
configproto.gpu_options.allow_growth = True
configproto.log_device_placement = args.log_device
configproto.allow_soft_placement = args.soft_placement
configproto.inter_op_parallelism_threads = args.num_cores
configproto.intra_op_parallelism_threads = args.num_cores
with tf.Graph().as_default(), tf.Session(config=configproto) as sess:
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE) if show_run_meta else None
run_metadata = tf.RunMetadata() if show_run_meta else None
model = graph_moudle.Model()
model.init_global_step()
vt, vs, vo = model.model_setup()
tf.initialize_all_variables().run()
cnt = 0
for var in vt:
cnt += 1
str_line = str(cnt) + '. ' + str(var.name) + ': ' + str(
var.get_shape())
print(str_line)
ssll = raw_input('aaaaa')
np.random.seed(1234567890)
qs = np.random.randint(0, args.vocab_size,
[10, args.batchsize, args.max_sent_length])
qsm = np.ones_like(qs, dtype=np.float32)
#qsm[:, :, -1:] = 0
ts = np.random.randint(0, args.vocab_size,
[10, args.batchsize, 2, args.max_sent_length])
tsm = np.ones_like(ts, dtype=np.float32)
url = np.ones([10, args.batchsize, 2], dtype=np.int32)
kw = np.ones([10, args.batchsize, 2, 10], dtype=np.int32)
kwm = np.ones([10, args.batchsize, 2, 10], dtype=np.float32)
g = np.random.randint(1, 3, (10, args.batchsize))
for i in range(400):
bs = i % 10
if bs == 0:
idx = np.random.shuffle(np.arange(10 * args.batchsize))
qs = np.reshape(qs, [10 * args.batchsize, -1])[idx]
ts = np.reshape(ts, [10 * args.batchsize, -1])[idx]
g = np.reshape(g, [10 * args.batchsize, -1])[idx]
qs = np.reshape(qs, qsm.shape)
ts = np.reshape(ts, tsm.shape)
stime = time.time()
loss, regu_loss = 0, 0
step, loss, pair_loss, regu_loss, acc, acc01, score, _ = \
model.run_epoch(sess, [qs[bs], qsm[bs], ts[bs], tsm[bs], url[bs], kw[bs], kwm[bs]], True,
run_options=run_options, run_metadata=run_metadata)
'''
pair_loss, acc, acc01, score = \
model.run_epoch(sess,
[qs[bs], qsm[bs], ts[bs], tsm[bs], np.ones([args.batchsize, 100], dtype=np.int32)],
False,
run_options=run_options, run_metadata=run_metadata)
'''
print(loss, pair_loss, regu_loss, acc, acc01, time.time() - stime)
#print(score[0, :], g[bs][0])
if show_run_meta:
tl = timeline.Timeline(run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format(show_memory=True)
with open(args.log_dir_path + '/timeline.json', 'w') as f:
f.write(ctf)
def main():
parseArgs(args)
args.log_dir_path = args.output + os.path.sep + 'test_' + os.path.split(
args.checkpoint)[1]
args.log_prefix = args.a0_model_name + '_'
if not os.path.exists(args.log_dir_path):
os.makedirs(args.log_dir_path)
if args.save_graph:
args.message("Resize batchsize to 1 for serving...")
if args.extra_wdict is not None:
args.wdict_path = args.extra_wdict
args.testfnms = [args.extra_testfnms]
args.message("Loading extra word dictionary from " + args.wdict_path)
configproto = tf.ConfigProto()
configproto.gpu_options.allow_growth = True
configproto.allow_soft_placement = True
args.batchsize = 400
test_batchsize = args.batchsize
#test_batchsize = 1
with tf.Graph().as_default(), tf.Session(config=configproto) as sess:
model = graph_moudle.Model()
model.init_global_step()
vt, vs, vo = model.model_setup()
tf.initialize_all_variables().run()
args.message('Loading trained model ' + str(args.checkpoint))
model.saver.restore(sess, args.checkpoint)
args.write_variables(vt)
if args.save_graph:
if args.save_only_trainable:
exporter_saver = tf.train.Saver(
var_list=tf.trainable_variables(), sharded=True)
else:
exporter_saver = tf.train.Saver(sharded=True)
online_feed_dict = {
'q': model.query_sent_in,
'qm': model.query_sent_mask,
't': model.title_sent_in,
'tm': model.title_sent_mask,
'is_training': model.is_training,
'keep_prob': model.keep_prob
}
online_fetch_dict = {'score': model.score}
model_exporter = exporter.Exporter(exporter_saver)
model_exporter.init(
sess.graph.as_graph_def(),
named_graph_signatures={
'inputs': exporter.generic_signature(online_feed_dict),
'outputs': exporter.generic_signature(online_fetch_dict)
})
model_exporter.export(args.log_dir_path, tf.constant(0), sess)
args.message("Successfully export graph to path:" +
args.log_dir_path)
#tf.train.write_graph(sess.graph_def, args.log_dir_path, 'graph.pbtxt', False)
return
args.write_args(args)
for test_file in args.testfnms:
print('Test ' + str(test_file))
f = open(test_file)
pairs = f.readlines()
n_test = len(pairs)
n_batches = int(n_test / test_batchsize)
if n_test % test_batchsize != 0:
n_batches += 1
tpair_loss, tacc, tacc01 = 0.0, 0.0, 0.0
for i in range(n_batches):
q, qm, t, tm, g = model.data_proc(
pairs[i * test_batchsize:(i + 1) * test_batchsize])
pair_loss, acc, acc01, score = \
model.run_epoch(sess, [q, qm, t, tm, g, False])
tpair_loss, tacc, tacc01 = tpair_loss + pair_loss, tacc + acc, tacc01 + acc01
out_str = "%f %f %f" % (pair_loss, acc, acc01)
for j in range(0, len(score)):
if i * test_batchsize + j >= n_test:
break
pair_line = pairs[i * test_batchsize + j].rstrip()
pair_tokens = pair_line.split('\t')
if len(pair_tokens) == 3:
score_str1 = pair_tokens[0] + '\n' + pair_tokens[
1] + '\n' + pair_tokens[2] + '\n' + str(
score[j][0]) + '\t' + str(score[j][1]) + '\n\n'
#score_str2 = pair_tokens[0] + '\t' + pair_tokens[2] + '\t' + str(score[j][1]) + '\n'
#score_str = pair_line + '\t' + str(score[j][0])+'-'+str(score[j][1]) + '\n'
sys.stderr.write(score_str1)
#sys.stderr.write(score_str2)
#print(out_str)
out_str = "Test " + str(
test_file) + " with checkpoint " + args.checkpoint
args.message(out_str)
n_batches = float(n_batches)
out_str = "pair loss:%f acc:%f acc01:%f" \
% (tpair_loss / n_batches, tacc / n_batches, tacc01 / n_batches)
args.message(out_str)
f.close()
from dev.config.final_config import * from dev.model.final_model import graph_moudle data_file_path = '/data/webrank/baidu_method.v1/demo' model = graph_moudle.Model() data_proc_fun = model.data_proc f = open(data_file_path) pairs = f.readlines() q, qm, t, tm, g = data_proc_fun(pairs) print(q[0]) print(qm[0]) print(t[0])
def main():
parseArgs(args)
args.log_dir_path = args.output + os.path.sep + 'test_' + os.path.split(
args.checkpoint)[1]
args.log_prefix = args.a0_model_name + '_'
if not os.path.exists(args.log_dir_path):
os.makedirs(args.log_dir_path)
if args.extra_wdict is not None:
args.wdict_path = args.extra_wdict
args.testfnms = [args.extra_testfnms]
args.message("Loading extra word dictionary from " + args.wdict_path)
configproto = tf.ConfigProto()
configproto.gpu_options.allow_growth = True
configproto.allow_soft_placement = True
args.batchsize = 400
test_batchsize = args.batchsize
with tf.Graph().as_default(), tf.Session(config=configproto) as sess:
model = graph_moudle.Model()
model.init_global_step()
vt, vs, vo = model.model_setup()
tf.initialize_all_variables().run()
args.message('Loading trained model ' + str(args.checkpoint))
model.saver.restore(sess, args.checkpoint)
args.write_variables(vt)
args.write_args(args)
for test_file in args.testfnms:
print('Test ' + str(test_file))
f = open(test_file)
pairs = f.readlines()
n_test = len(pairs)
n_batches = int(n_test / test_batchsize)
if n_test % test_batchsize != 0:
n_batches += 1
for i in range(n_batches):
data_proc_slice = model.data_proc(
pairs[i * test_batchsize:(i + 1) * test_batchsize],
n_samples=1)
score = model.run_score(sess, data_proc_slice)[0]
for j in range(0, len(score)):
if i * test_batchsize + j >= n_test:
break
pair_line = pairs[i * test_batchsize + j].rstrip()
pair_tokens = pair_line.split('\t')
if len(pair_tokens) <= 9:
score_str1 = pair_tokens[0] + '\t' + pair_tokens[
1] + '\t' + pair_tokens[2] + '\t' + str(
score[j][0]) + '\n'
else:
score_str1 = pair_tokens[0] + '\t' + pair_tokens[
3] + '\t' + pair_tokens[4] + '\t' + str(
score[j][0]) + '\t' + pair_tokens[-1] + '\n'
sys.stderr.write(score_str1)
#print(out_str)
out_str = "Print log of " + str(
test_file
) + " with checkpoint " + args.checkpoint + " finished!"
args.message(out_str)
f.close()
def main():
parseArgs(args)
args.log_dir_path = args.output + os.path.sep + 'test_' + os.path.split(args.checkpoint)[1]
args.log_prefix = args.a0_model_name + '_'
if not os.path.exists(args.log_dir_path):
os.makedirs(args.log_dir_path)
# if args.save_graph:
# args.message("Resize batchsize to 1 for serving...")
#
# if args.extra_wdict is not None:
# args.wdict_path = args.extra_wdict
# args.testfnms = [args.extra_testfnms]
# args.message("Loading extra word dictionary from " + args.wdict_path)
# config.gpu_options.per_process_gpu_memory_fraction = 0.9 # 占用GPU90%的显存
configproto = tf.ConfigProto()
configproto.gpu_options.allow_growth = True
configproto.allow_soft_placement = True
test_batchsize = args.batchsize
ops.reset_default_graph()
with tf.Graph().as_default(), tf.Session(config=configproto) as sess:
model = graph_moudle.Model()
model.init_global_step()
vt, vs, vo = model.model_setup()
tf.global_variables_initializer().run()
args.message('Loading trained model ' + str(args.checkpoint))
model.saver.restore(sess, args.checkpoint)
args.write_variables(vt)
args.write_args(args)
fw = open(args.predit_output,"w")
for test_file in args.testfnms:
print('Test ' + str(test_file))
f = open(test_file)
pairs = read_input(f.readlines())
n_test = len(pairs)
n_batches = int(n_test / test_batchsize)
if n_test % test_batchsize != 0:
n_batches += 1
for i in range(n_batches):
result = ""
data_proc_slice = model.data_proc(pairs[i * test_batchsize: (i + 1) * test_batchsize])
label, q, qm, t, tm, dm, _ = data_proc_slice
out_str = "q: %f - %f - %f - %f - %f" % (q[0][0], q[0][1], q[0][2], q[0][3], q[0][4])
args.message(out_str, True)
out_str = "qm: %f - %f - %f - %f - %f" % (qm[0][0], qm[0][1], qm[0][2], qm[0][3], qm[0][4])
args.message(out_str, True)
out_str = "t: %f - %f - %f - %f - %f" % (t[0][0][0], t[0][0][1], t[0][0][2], t[0][0][3], t[0][0][4])
args.message(out_str, True)
out_str = "qm: %f - %f - %f - %f - %f" % (tm[0][0][0], tm[0][0][1], tm[0][0][2], tm[0][0][3], tm[0][0][4])
args.message(out_str, True)
out_str = "label: %f - %f - %f - %f - %f" % (label[0], label[1], label[2], label[3], label[4])
args.message(out_str, True)
acc, score = \
model.run_epoch(sess, data_proc_slice, False)
print "acc: " + str(acc)
for j in range(0, min(len(pairs)-(i*test_batchsize), test_batchsize)):
pair_line = pairs[i * test_batchsize + j].rstrip()
result = result + str(round(score[j],2)) + '\t' + pair_line + '\n'
fw.write(result)
fw.close()
def main():
parseArgs(args)
args.log_dir_path = args.output + os.path.sep + 'test_' + os.path.split(
args.checkpoint)[1]
args.log_prefix = args.a0_model_name + '_'
if not os.path.exists(args.log_dir_path):
os.makedirs(args.log_dir_path)
if args.save_graph is not None:
args.message("Resize batchsize to 1 for serving...")
if args.extra_wdict is not None:
args.wdict_path = args.extra_wdict
args.testfnms = [args.extra_testfnms]
args.message("Loading extra word dictionary from " + args.wdict_path)
configproto = tf.ConfigProto()
configproto.gpu_options.allow_growth = True
configproto.allow_soft_placement = True
args.batchsize = 400
test_batchsize = args.batchsize
#test_batchsize = 1
with tf.Graph().as_default(), tf.Session(config=configproto) as sess:
model = graph_moudle.Model()
model.init_global_step()
vt, vs, vo = model.model_setup()
tf.initialize_all_variables().run()
args.message('Loading trained model ' + str(args.checkpoint))
model.saver.restore(sess, args.checkpoint)
args.write_variables(vt)
if args.save_graph:
if args.save_only_trainable:
exporter_saver = tf.train.Saver(
var_list=tf.trainable_variables(), sharded=True)
else:
exporter_saver = tf.train.Saver(sharded=True)
online_feed_dict = {
'q': model.query_sent_in,
'qm': model.query_sent_mask,
't': model.title_sent_in,
'tm': model.title_sent_mask,
'is_training': model.is_training,
'keep_prob': model.keep_prob
}
online_fetch_dict = {'score': model.score}
model_exporter = exporter.Exporter(exporter_saver)
model_exporter.init(
sess.graph.as_graph_def(),
named_graph_signatures={
'inputs': exporter.generic_signature(online_feed_dict),
'outputs': exporter.generic_signature(online_fetch_dict)
})
model_exporter.export(args.log_dir_path, tf.constant(0), sess)
args.message("Successfully export graph to path:" +
args.log_dir_path)
#tf.train.write_graph(sess.graph_def, args.log_dir_path, 'graph.pbtxt', False)
return
args.write_args(args)
fw = open(args.predit_output, "w")
fw2 = open("./data/detail.txt", "w")
for test_file in args.testfnms:
print('Test ' + str(test_file))
f = open(test_file)
data = read_input(f.readlines())
for key, kviter in groupby(data, itemgetter(2)):
pairs = []
itemNum = 0
for k in kviter:
#if(len(k[0])>15 and len(k[1])>15):
pairs.append("\t".join(k))
itemNum = itemNum + 1
if itemNum > 20:
break
if (itemNum < 5):
continue
n_test = len(pairs)
n_batches = int(n_test / test_batchsize)
if n_test % test_batchsize != 0:
n_batches += 1
tpair_loss, tacc, tacc01 = 0.0, 0.0, 0.0
score_sum = 0
result = ""
detail = ""
for i in range(n_batches):
q, qm, t, tm, g = model.data_proc(
pairs[i * test_batchsize:(i + 1) * test_batchsize])
pair_loss, acc, acc01, score = \
model.run_epoch(sess, [q, qm, t, tm, g, False])
for j in range(0, itemNum):
if i * test_batchsize + j >= n_test:
break
pair_line = pairs[i * test_batchsize + j].rstrip()
pair_tokens = pair_line.split('\t')
if len(pair_tokens) > 2:
score_sum = score_sum + score[j][0]
score_str1 = key + '\t' + pair_tokens[
0] + '\t' + pair_tokens[1] + '\t' + str(
score[j][0]) + '\n'
detail = detail + score_str1
# fw2.write(score_str1)
score_avg = score_sum / itemNum
result = result + key + '\t' + str(score_avg) + '\n'
fw2.write(detail)
fw.write(result)
f.close()
fw.close()
fw2.close()
