def analyze_unk():
corruption_prob = 0.06
print "Loading data using vocab size %d..." % FLAGS.vocab_size
word2id = data_utils.load_from_dump(FLAGS.data_dir +
'%d.vocab' % FLAGS.vocab_size)
train_loader = data_utils.DataLoader(FLAGS.data_dir +
'train.vocab%d.id' % FLAGS.vocab_size,
50,
FLAGS.sent_len,
unk_prob=corruption_prob)
dev_loader = data_utils.DataLoader(
FLAGS.data_dir + 'dev.vocab%d.id' % FLAGS.vocab_size, 50,
FLAGS.sent_len)
test_loader = data_utils.DataLoader(
FLAGS.data_dir + 'test.vocab%d.id' % FLAGS.vocab_size, 50,
FLAGS.sent_len)
print "Counting..."
train_unk, train_total = get_unk_count_in_dataset(train_loader)
dev_unk, dev_total = get_unk_count_in_dataset(dev_loader)
test_unk, test_total = get_unk_count_in_dataset(test_loader)
print "Training token count:"
print "\tunk:%d\ttotal:%d\tratio:%g" % (train_unk, train_total,
1.0 * train_unk / train_total)
print "Dev token count:"
print "\tunk:%d\ttotal:%d\tratio:%g" % (dev_unk, dev_total,
1.0 * dev_unk / dev_total)
print "Test token count:"
print "\tunk:%d\ttotal:%d\tratio:%g" % (test_unk, test_total,
1.0 * test_unk / test_total)
def test():
if not os.path.exists(args.model_path):
os.mkdir(args.model_path)
print "load mode from : ", args.model_path
dump_path = os.path.join(args.data_path,
"syncompqa_data_zero_%d.pkl" % args.vocab_min_frq)
if not os.path.exists(dump_path):
dataloader = data_utils.DataLoader(args.data_path, args.vocab_min_frq)
cPickle.dump(dataloader, open(dump_path, "wb"))
else:
dataloader = cPickle.load(open(dump_path, "rb"))
print "test instance: ", len(dataloader.test_data)
print "vocab size: ", dataloader.vocab_size
model = models.CompQAModel(dataloader.vocab_size, args.embedding_size,
dataloader.max_q_len, dataloader.max_a_len,
dataloader.max_fact_num, args.state_size,
args.num_layers, args.num_samples,
args.max_gradient_norm, False, args.cell,
args.optimizer, args.learning_rate)
print "begining to test model"
model.test(dataloader, args.model_path)
pass
def main():
if not os.path.exists(args.model_path):
os.mkdir(args.model_path)
print "save mode to : ", args.model_path
dump_path = os.path.join(args.data_path,
"syncompqa_data_%d.pkl" % args.vocab_min_frq)
if not os.path.exists(dump_path):
dataloader = data_utils.DataLoader(args.data_path, args.vocab_min_frq)
cPickle.dump(dataloader, open(dump_path, "wb"))
else:
dataloader = cPickle.load(open(dump_path, "rb"))
print "train instance: ", len(dataloader.train_data)
print "vocab size: ", dataloader.vocab_size
print("start to build the model .. ")
model = models.CompQAModel(state_size=args.state_size,
num_layers=args.num_layers,
num_samples=args.num_samples,
max_grad_norm=args.max_gradient_norm,
is_train=True,
cell_type=args.cell,
optimizer_name=args.optimizer,
learning_rate=args.learning_rate)
# model = models.CompQAModel(dataloader.vocab_size, args.embedding_size,
# dataloader.max_q_len, dataloader.max_a_len,
# dataloader.max_fact_num,
# args.state_size, args.num_layers, args.num_samples,
# args.max_gradient_norm, True, args.cell,
# args.optimizer, args.learning_rate)
print "begining to train model .. "
model.fit(dataloader, args.batch_size, args.epoch_size,
args.checkpoint_step, args.model_path)
def main():
if not os.path.exists(args.model_path):
os.mkdir(args.model_path)
print "save mode to : ", args.model_path
dump_path = os.path.join(args.data_path, "data.pkl")
if not os.path.exists(dump_path):
dataloader = data_utils.DataLoader(args.data_path)
cPickle.dump(dataloader, open(dump_path, "wb"))
else:
dataloader = cPickle.load(open(dump_path, "rb"))
print "train instance: ", len(dataloader.train_data)
print "char size: ", dataloader.char_vocab_size
print "tag size: ", dataloader.tag_vocab_size
model = models.BiLSTM_CRF(dataloader.char_vocab_size,
dataloader.tag_vocab_size,
dataloader.max_sent_len, None,
args.embedding_size, args.hidden_size,
args.num_layers, True, True)
print "begining to train model"
model.train(dataloader, args.batch_size, args.epoch_size,
args.checkpoint_step, args.model_path)
def make_loaders(opt):
"""makes training/val/test"""
batch_size = opt.batch_size * opt.world_size
eval_batch_size = opt.eval_batch_size * opt.world_size
seq_length = opt.seq_length
if seq_length < 0:
seq_length = seq_length * opt.world_size
eval_seq_length = opt.eval_seq_length
if opt.eval_seq_length < 0:
eval_seq_length = eval_seq_length * opt.world_size
# TODO: fix data race in lazy loader
# data_loader_args = {'num_workers': 10, 'shuffle': opt.shuffle, 'batch_size': batch_size,
data_loader_args = {
'num_workers': 1,
'shuffle': opt.shuffle,
'batch_size': batch_size,
'pin_memory': True,
'transpose': opt.transpose,
'distributed': opt.world_size > 1,
'rank': opt.rank,
'world_size': opt.world_size,
'drop_last': opt.world_size > 1
}
split = get_split(opt)
data_set_args = {
'path': opt.data,
'seq_length': seq_length,
'lazy': opt.lazy,
'text_key': opt.text_key,
'label_key': opt.label_key,
'preprocess': opt.preprocess,
'persist_state': opt.persist_state,
'delim': opt.delim,
'num_shards': opt.num_shards,
'ds_type': opt.data_set_type,
'split': split,
'loose': opt.loose_json
}
eval_loader_args = copy.copy(data_loader_args)
eval_set_args = copy.copy(data_set_args)
eval_set_args['split'] = [1.]
# if optional eval args were set then replace their equivalent values in the arg dict
if opt.eval_batch_size != 0:
eval_loader_args['batch_size'] = eval_batch_size
if opt.eval_seq_length != 0:
eval_set_args['seq_length'] = eval_seq_length
if opt.eval_text_key != 'None':
eval_set_args['text_key'] = opt.eval_text_key
if opt.eval_label_key != 'None':
eval_set_args['label_key'] = opt.eval_label_key
train = None
valid = None
test = None
if opt.data != 'None':
train = data_utils.make_dataset(**data_set_args)
if should_split(split):
train, valid, test = train
if opt.valid != 'None':
eval_set_args['path'] = opt.valid
valid = data_utils.make_dataset(**eval_set_args)
if test is None and opt.test != 'None':
eval_set_args['path'] = opt.test
test = data_utils.make_dataset(**eval_set_args)
if train is not None and opt.batch_size > 0:
train = data_utils.DataLoader(train, **data_loader_args)
if valid is not None:
if opt.data_set_type == 'unsupervised':
if opt.eval_seq_length != 0:
valid.set_seq_len(eval_seq_length)
if opt.val_shards != 0:
valid.set_num_shards(opt.val_shards)
valid = data_utils.DataLoader(valid, **eval_loader_args)
if test is not None:
if opt.data_set_type == 'unsupervised':
if opt.eval_seq_length != 0:
test.set_seq_len(eval_seq_length)
if opt.test_shards != 0:
test.set_num_shards(opt.test_shards)
test = data_utils.DataLoader(test, **eval_loader_args)
return train, valid, test
def train():
# print training info
print _get_training_info()
# dealing with files
print "Loading data from files..."
train_loader = data_utils.DataLoader(
os.path.join(FLAGS.data_dir, 'train.vocab%d.id' % FLAGS.vocab_size),
FLAGS.batch_size,
FLAGS.sent_len,
subsample=FLAGS.subsample,
unk_prob=FLAGS.corrupt_rate
) # use a subsample of the data if specified
# load cv dataset
dev_loaders = []
test_loaders = []
for i in range(100):
dev_loader = data_utils.DataLoader(
os.path.join(FLAGS.data_dir, 'cv',
'dev.vocab%d.id.%d' % (FLAGS.vocab_size, i)),
FLAGS.batch_size, FLAGS.sent_len)
test_loader = data_utils.DataLoader(
os.path.join(FLAGS.data_dir, 'cv',
'test.vocab%d.id.%d' % (FLAGS.vocab_size, i)),
FLAGS.batch_size, FLAGS.sent_len)
dev_loaders.append(dev_loader)
test_loaders.append(test_loader)
max_steps = train_loader.num_batches * FLAGS.num_epoch
print "# Examples in training data:"
print train_loader.num_examples
# load label2id mapping and create inverse mapping
label2id = data_utils.LABEL_TO_ID
id2label = dict([(v, k) for k, v in label2id.iteritems()])
key = random.randint(1e5, 1e6 - 1) # get a random 6-digit int
test_key_file_list = []
test_prediction_file_list = []
dev_key_file_list = []
dev_prediction_file_list = []
for i in range(100):
test_key_file = os.path.join(
FLAGS.train_dir,
str(key) + '.shuffled.test.key.tmp.%d' % i)
test_prediction_file = os.path.join(
FLAGS.train_dir,
str(key) + '.shuffled.test.prediction.tmp.%d' % i)
dev_key_file = os.path.join(FLAGS.train_dir,
str(key) + '.shuffled.dev.key.tmp.%d' % i)
dev_prediction_file = os.path.join(
FLAGS.train_dir,
str(key) + '.shuffled.dev.prediction.tmp.%d' % i)
test_key_file_list.append(test_key_file)
test_prediction_file_list.append(test_prediction_file)
dev_key_file_list.append(dev_key_file)
dev_prediction_file_list.append(dev_prediction_file)
test_loaders[i].write_keys(test_key_file, id2label=id2label)
dev_loaders[i].write_keys(dev_key_file, id2label=id2label)
with tf.Graph().as_default():
print "Constructing model %s..." % (FLAGS.model)
with tf.variable_scope('model', reuse=None):
m = _get_model(is_train=True)
with tf.variable_scope('model', reuse=True):
mdev = _get_model(is_train=False)
saver = tf.train.Saver(tf.all_variables(), max_to_keep=2)
save_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
config = tf.ConfigProto()
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_mem, allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(device_count={"GPU": 1},
gpu_options=gpu_options))
summary_writer = tf.train.SummaryWriter(FLAGS.train_dir,
graph=sess.graph)
sess.run(tf.initialize_all_variables())
if FLAGS.use_pretrain:
print "Use pretrained embeddings to initialize model ..."
emb_file = os.path.join(
FLAGS.data_dir,
"emb-v%d-d%d.npy" % (FLAGS.vocab_size, FLAGS.hidden_size))
if not os.path.exists(emb_file):
raise Exception("Pretrained vector file does not exist at: " +
emb_file)
pretrained_embedding = np.load(emb_file)
m.assign_embedding(sess, pretrained_embedding)
current_lr = FLAGS.init_lr
global_step = 0
training_history = []
dev_f_history = []
test_f_history = []
best_dev_scores = []
best_test_scores = []
def eval_once(mdev, sess, data_loader):
data_loader.reset_pointer()
predictions = []
confidences = []
dev_loss = 0.0
for _ in xrange(data_loader.num_batches):
x_batch, y_batch, x_lens = data_loader.next_batch()
feed = _get_feed_dict(mdev,
x_batch,
y_batch,
x_lens,
use_pos=(FLAGS.pos_size > 0),
use_ner=(FLAGS.ner_size > 0),
use_deprel=(FLAGS.deprel_size > 0))
loss_value, pred, conf = sess.run(
[mdev.loss, mdev.prediction, mdev.confidence],
feed_dict=feed)
predictions += list(pred)
confidences += list(conf)
dev_loss += loss_value
dev_loss /= data_loader.num_batches
return dev_loss, predictions, confidences
print "Start training with %d epochs, and %d steps per epoch..." % (
FLAGS.num_epoch, train_loader.num_batches)
for epoch in xrange(FLAGS.num_epoch):
train_loss = 0.0
train_loader.reset_pointer()
m.assign_lr(sess, current_lr)
for _ in xrange(train_loader.num_batches):
global_step += 1
start_time = time.time()
x_batch, y_batch, x_lens = train_loader.next_batch()
feed = _get_feed_dict(m,
x_batch,
y_batch,
x_lens,
use_pos=(FLAGS.pos_size > 0),
use_ner=(FLAGS.ner_size > 0),
use_deprel=(FLAGS.deprel_size > 0))
_, loss_value = sess.run([m.train_op, m.loss], feed_dict=feed)
duration = time.time() - start_time
train_loss += loss_value
assert not np.isnan(loss_value), "Model loss is NaN."
if global_step % FLAGS.log_step == 0:
format_str = (
'%s: step %d/%d (epoch %d/%d), loss = %.6f (%.3f sec/batch), lr: %.6f'
)
print format_str % (datetime.now(), global_step, max_steps,
epoch + 1, FLAGS.num_epoch, loss_value,
duration, current_lr)
# summary loss after each epoch
train_loss /= train_loader.num_batches
summary_writer.add_summary(_summary_for_scalar(
'eval/training_loss', train_loss),
global_step=epoch)
# do CV on test set and use average score
avg_dev_loss = 0.0
avg_test_loss = 0.0
avg_dev_f = 0.0
avg_dev_p = 0.0
avg_dev_r = 0.0
avg_test_f = 0.0
avg_test_p = 0.0
avg_test_r = 0.0
for i in range(100):
dev_loss, dev_preds, dev_confs = eval_once(
mdev, sess, dev_loaders[i])
avg_dev_loss += dev_loss
summary_writer.add_summary(_summary_for_scalar(
'eval/dev_loss%d' % i, dev_loss),
global_step=epoch)
_write_prediction_file(dev_preds, dev_confs, id2label,
dev_prediction_file_list[i])
# print "Evaluating on dev set..."
dev_prec, dev_recall, dev_f = scorer.score(
dev_key_file_list[i], [dev_prediction_file_list[i]],
FLAGS.f_measure)
avg_dev_f += dev_f
avg_dev_p += dev_prec
avg_dev_r += dev_recall
test_loss, test_preds, test_confs = eval_once(
mdev, sess, test_loaders[i])
avg_test_loss += test_loss
summary_writer.add_summary(_summary_for_scalar(
'eval/test_loss%d' % i, test_loss),
global_step=epoch)
_write_prediction_file(test_preds, test_confs, id2label,
test_prediction_file_list[i])
# print "Evaluating on test set..."
test_prec, test_recall, test_f = scorer.score(
test_key_file_list[i], [test_prediction_file_list[i]],
FLAGS.f_measure)
avg_test_f += test_f
avg_test_p += test_prec
avg_test_r += test_recall
avg_dev_loss /= 100
avg_test_loss /= 100
avg_dev_f /= 100
avg_dev_p /= 100
avg_dev_r /= 100
avg_test_f /= 100
avg_test_p /= 100
avg_test_r /= 100
print "Epoch %d: training_loss = %.6f" % (epoch + 1, train_loss)
print "Epoch %d: avg_dev_loss = %.6f, avg_dev_f-%g = %.6f" % (
epoch + 1, avg_dev_loss, FLAGS.f_measure, avg_dev_f)
print "Epoch %d: avg_test_loss = %.6f, avg_test_f-%g = %.6f" % (
epoch + 1, avg_test_loss, FLAGS.f_measure, avg_test_f)
# decrease learning rate if dev_f does not increase after an epoch
if len(dev_f_history) > 10 and avg_dev_f <= dev_f_history[-1]:
current_lr *= FLAGS.lr_decay
training_history.append(train_loss)
# save the model when best f score is achieved on dev set
if len(dev_f_history) == 0 or (len(dev_f_history) > 0
and avg_dev_f > max(dev_f_history)):
saver.save(sess, save_path, global_step=epoch)
print "\tmodel saved at epoch %d, with best dev dataset f-%g score %.6f" % (
epoch + 1, FLAGS.f_measure, avg_dev_f)
best_dev_scores = [avg_dev_p, avg_dev_r, avg_dev_f]
best_test_scores = [avg_test_p, avg_test_r, avg_test_f]
dev_f_history.append(avg_dev_f)
test_f_history.append(avg_test_f)
# stop learning if lr is too low
if current_lr < 1e-6:
break
# saver.save(sess, save_path, global_step=epoch)
print "Training ended with %d epochs." % epoch
print "\tBest dev scores achieved (P, R, F-%g):\t%.3f\t%.3f\t%.3f" % tuple(
[FLAGS.f_measure] + [x * 100 for x in best_dev_scores])
print "\tBest test scores achieved on best dev scores (P, R, F-%g):\t%.3f\t%.3f\t%.3f" % tuple(
[FLAGS.f_measure] + [x * 100 for x in best_test_scores])
# clean up
for dev_key_file, dev_prediction_file, test_key_file, test_prediction_file in zip(
dev_key_file_list, dev_prediction_file_list, test_key_file_list,
test_prediction_file_list):
if os.path.exists(dev_key_file):
os.remove(dev_key_file)
if os.path.exists(dev_prediction_file):
os.remove(dev_prediction_file)
if os.path.exists(test_key_file):
os.remove(test_key_file)
if os.path.exists(test_prediction_file):
os.remove(test_prediction_file)
if __name__ == '__main__':
import data_utils
import runners
import utils
_BATCH_SIZE = 2
_MAX_LENGTH = 20
vocab_path = 'sample_data/sample_vocab.txt'
train_source_path = 'sample_data/sample_source.txt'
train_target_path = 'sample_data/sample_target.txt'
test_source_path = 'sample_data/sample_source.txt'
test_target_path = 'sample_data/sample_target.txt'
train_data_loader = data_utils.DataLoader(vocab_path)
train_data_loader.load_data_info(train_source_path,
train_target_path,
batch_size=_BATCH_SIZE,
max_length=_MAX_LENGTH)
test_data_loader = data_utils.DataLoader(vocab_path)
test_data_loader.load_data_info(train_source_path,
train_target_path,
batch_size=_BATCH_SIZE,
max_length=_MAX_LENGTH)
config_dict = {
'source_vocab_size': train_data_loader.indexer.size(),
'target_vocab_size': train_data_loader.indexer.size(),
'num_layers': 4,
'model_size': 16,
def evaluate():
g = tf.Graph()
with g.as_default():
# load data get iterator
data_loader = data_utils.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE,
NUM_EPOCHS)
iterator = data_loader.load_data(VALID_TFR_PATH, False)
frameNo, image, label = iterator.get_next()
# VGG FACE network
VGGFace_network = vgg_face.VGGFace(SEQUENCE_LENGTH * BATCH_SIZE)
image_batch = tf.reshape(image, [-1, 96, 96, 3])
VGGFace_network.setup(
image_batch
) # image_batch is a tensor of shape (batch_size*seq_length,image_dim,image_dim,3)
face_output = VGGFace_network.get_face_fc0()
# RNN part
rnn_in = reshape_to_rnn(face_output)
prediction = models.get_prediction(rnn_in)
prediction = tf.reshape(prediction, [-1, 2])
label_batch = tf.reshape(label, [-1, 2])
with tf.Session(graph=g) as sess:
# if not os.path.exists(SUMMARY_PATH):
# os.makedirs(SUMMARY_PATH)
# eval_summary_writer = tf.summary.FileWriter(SUMMARY_PATH, graph=g)
restore_variables(sess)
total_ccc_v = 0
total_ccc_a = 0
total_ccc = 0
total_mse_v = 0
total_mse_a = 0
total_mse = 0
for i in range(NUM_BATCHES):
try:
pred, lab = sess.run([prediction, label_batch])
except tf.errors.OutOfRangeError:
break
print 'prediction batch : ' + str(i)
print pred
conc_arousal = concordance_cc2(pred[:, 1], lab[:, 1])
conc_valence = concordance_cc2(pred[:, 0], lab[:, 0])
mse_arousal = sum((pred[:, 1] - lab[:, 1])**2) / len(lab[:, 1])
mse_valence = sum((pred[:, 0] - lab[:, 0])**2) / len(lab[:, 0])
total_ccc_v += conc_valence
total_ccc_a += conc_arousal
total_ccc += ((conc_valence + conc_valence) / 2.0)
total_mse_v += (mse_valence)
total_mse_a += (mse_arousal)
total_mse += ((mse_arousal + mse_valence) / 2.0)
print 'Finish read data'
# add summary
num_batches = float(NUM_BATCHES)
# summary = tf.Summary()
# summary.value.add(tag='eval/conc_valence', simple_value=float(total_ccc_v/num_batches))
# summary.value.add(tag='eval/conc_arousal', simple_value=float(total_ccc_a/num_batches))
# summary.value.add(tag='eval/conc_total', simple_value=float(total_ccc/num_batches))
# summary.value.add(tag='eval/mse_arousal', simple_value=float(total_mse_a/num_batches))
# summary.value.add(tag='eval/mse_valence', simple_value=float(total_mse_v/num_batches))
#
# eval_summary_writer.add_summary(summary, get_global_step(model_path))
print '#####################Summary#######################'
# print 'Evaluate model {}'.format(model_path)
print 'Concordance on valence : {}'.format(
float(total_ccc_v / num_batches))
print 'Concordance on arousal : {}'.format(
float(total_ccc_a / num_batches))
print 'Concordance on total : {}'.format(
float(total_ccc / num_batches))
print 'MSE Arousal : {}'.format(float(total_mse_a / num_batches))
print 'MSE Valence : {}'.format(float(total_mse_v / num_batches))
print 'MSE TOTAL : {}'.format(float(total_mse / num_batches))
def evaluate():
print "Building graph and loading model..."
with tf.Graph().as_default():
### the first model will be doing the full batches (a residual of examples will be left)
with tf.variable_scope('model'):
m = _get_model(is_train=False)
saver = tf.train.Saver(tf.all_variables())
config = tf.ConfigProto()
gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(device_count={"GPU": 1},
gpu_options=gpu_options))
ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
else:
raise IOError("Loading checkpoint file failed!")
print "====> Evaluating on %s data" % FLAGS.eval_set
print "Loading %s data..." % FLAGS.eval_set
loader = data_utils.DataLoader(
os.path.join(FLAGS.data_dir,
'%s.vocab%d.id' % (FLAGS.eval_set, FLAGS.vocab_size)),
FLAGS.batch_size, FLAGS.sent_len
) # load test data with batch_size 1; this is too slow
# load label2id mapping and create inverse mapping
label2id = data_utils.LABEL_TO_ID
id2label = dict([(v, k) for k, v in label2id.iteritems()])
# key = random.randint(1e5, 1e6-1) # get a random 6-digit int
test_key_file = os.path.join(FLAGS.train_dir,
'shuffled.%s.key.tmp' % FLAGS.eval_set)
test_prediction_file = os.path.join(
FLAGS.train_dir, 'shuffled.%s.prediction.tmp' % FLAGS.eval_set)
test_prob_file = os.path.join(FLAGS.train_dir,
'shuffled.%s.probs.tmp' % FLAGS.eval_set)
loader.write_keys(test_key_file,
id2label=id2label,
include_residual=True
) # write shuffled key to file, used by scorer
test_loss = .0
print "Evaluating on %d test examples with full batch..." % (
loader.num_batches * loader.batch_size)
preds, confs = [], []
all_probs = np.zeros([loader.num_examples, FLAGS.num_class])
for i in range(loader.num_batches):
x, y, x_lens = loader.next_batch()
feed = _get_feed_dict(m,
x,
y,
x_lens,
use_pos=(FLAGS.pos_size > 0),
use_ner=(FLAGS.ner_size > 0),
use_deprel=(FLAGS.deprel_size > 0))
loss_value, predictions, confidences, probs = sess.run(
[m.loss, m.prediction, m.confidence, m.probs], feed_dict=feed)
test_loss += loss_value
preds += list(predictions)
confs += list(confidences)
all_probs[i * loader.batch_size:(i + 1) *
loader.batch_size, :] = probs
### second model will do the residuals with one batch
if loader.num_residual > 0:
print "Evaluating on an residual of %d examples..." % loader.num_residual
x, y, x_lens = loader.get_residual()
feed = _get_feed_dict(m,
x,
y,
x_lens,
use_pos=(FLAGS.pos_size > 0),
use_ner=(FLAGS.ner_size > 0),
use_deprel=(FLAGS.deprel_size > 0))
loss_value, predictions, confidences, probs = sess.run(
[m.loss, m.prediction, m.confidence, m.probs], feed_dict=feed)
test_loss += loss_value
preds += list(predictions)
confs += list(confidences)
all_probs[loader.num_batches * loader.batch_size:, :] = probs
if not FLAGS.use_confidence:
confs = [1.0] * len(confs)
_write_prediction_file(preds, confs, all_probs, id2label,
test_prediction_file, test_prob_file)
test_loss /= loader.num_examples
print "%s: test_loss = %.6f" % (datetime.now(), test_loss)
prec, recall, f1 = scorer.score(test_key_file, [test_prediction_file],
verbose=True)
# clean up
if FLAGS.cleanup and os.path.exists(test_key_file):
os.remove(test_key_file)
if FLAGS.cleanup and os.path.exists(test_prediction_file):
os.remove(test_prediction_file)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense
import numpy as np
import data_utils
data_dim = 128
timesteps = 128
train_data = data_utils.load_training_data()
test_data = data_utils.load_test_data()
# loader = data_utils.DataLoader(data=data,batch_size=train_config.batch_size, num_steps=train_config.num_steps)
data_loader = data_utils.DataLoader(train_data, 7352, 1)
# x_test, y_test = data_utils.DataLoader(train_data, 128, 1).next_batch()
# expected input data shape: (batch_size, timesteps, data_dim)
model = Sequential()
model.add(
LSTM(256, return_sequences=True,
input_shape=(1, 1))) # returns a sequence of vectors of dimension 32
model.add(LSTM(
256,
return_sequences=True)) # returns a sequence of vectors of dimension 32
model.add(LSTM(
256, return_sequences=True)) # return a single vector of dimension 32
model.add(Dense(128, activation="sigmoid", name="DENSE1"))
model.add(Dense(72, activation="sigmoid", name="DENSE2"))
model.add(Dense(1, activation='softmax'))
def train():
g = tf.Graph()
with g.as_default():
with tf.Session(graph=g) as sess:
# build the graph
x = tf.placeholder(
tf.float32,
shape=[BATCH_SIZE, SEQUENCE_LENGTH, HEIGHT, WIDTH, CHANNELS])
y_ = tf.placeholder(tf.int16,
shape=[BATCH_SIZE, SEQUENCE_LENGTH, 2])
prediction = get_prediction(x)
total_loss = get_loss(prediction, y_)
my_global_step = tf.get_variable("global_step",
initializer=tf.constant(0),
trainable=False)
# restore or init variables
step = restore_all_variables(sess)
# ???????????????????????????
optimizer = tf.train.AdamOptimizer(LEARNING_RATE).minimize(
total_loss, global_step=step)
# prepare summary & saver
saver_model = tf.train.Saver(max_to_keep=10000)
summary_writer = tf.summary.FileWriter(TRAIN_DIR, graph=sess.graph)
merged_summaries = tf.summary.merge_all()
# get data iterator
data_loader = data_utils.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE,
NUM_EPOCHS)
iterator = data_loader.load_data(TRAIN_TFR_PATH, True)
frameNo, image, label = iterator.get_next()
for i in range(MAX_STEP):
try:
start_time = time.time()
frameNos, images, labels = sess.run(
[frameNo, image, label])
_, step_loss, step_summary = sess.run([optimizer],
feed_dict={
x: images,
y_: labels
})
time_step = time.time() - start_time
if step % 500 == 0 or (step + 1) == MAX_STEP:
step_loss, step_summary = sess.run(
[total_loss, merged_summaries],
feed_dict={
x: images,
y_: labels
})
summary_writer.add_summary(step_summary,
global_step=step)
write(
"Global_Step {}: loss = {:.4f} ({:.2f} sec/step)".
format(step, step_loss, time_step))
if step % 1500 == 0 or (step + 1) == MAX_STEP:
# save model to ckpts
saver_model.save(sess, MODEL_PATH, global_step=step)
step += 1
except tf.errors.OutOfRangeError:
break
summary_writer.close()
sess.close()
