def videoGetter(filename,
outdir='out/prepro/',
update_rate=50):
DEST_IMG_SIZE = (270, 480)
cap = VideoCapture(filename)
frameCount = int(cap.get(CAP_PROP_FRAME_COUNT))
frameWidth = int(cap.get(CAP_PROP_FRAME_WIDTH))
frameHeight = int(cap.get(CAP_PROP_FRAME_HEIGHT))
status = Progbar(frameCount, text=filename)
# Note that for some reason, cv2 reads images hieght and then width
buf = empty(
(frameCount, DEST_IMG_SIZE[1], DEST_IMG_SIZE[0], 3), dtype('int8'))
raw = empty((frameWidth, frameHeight, 3), dtype('uint8'))
middle = empty((DEST_IMG_SIZE[1], DEST_IMG_SIZE[0], 3), dtype('uint8'))
fc = 0
ret = True
while (fc < frameCount and ret):
ret, temp = cap.read()
middle[:, :, :] = resize(
raw, None, fx=0.25, fy=0.25, interpolation=INTER_AREA)
buf[fc] = (middle.astype('int8') - 255 // 2)
if fc % update_rate == 0:
status.update(fc)
fc += 1
cap.release()
del cap, raw, middle
filename = filename.rsplit('/', 1)[1]
outpath = outdir + filename.rsplit('.', 1)[0] + '.npy'
return save(outpath, buf)
def train_step(model, data, summary_op, summary_writer):
"""Train one epoch
"""
start_time = time.time()
sess = model.sess
runopts = tf.RunOptions(report_tensor_allocations_upon_oom=True)
prog = Progbar(target=data.num_batches)
iterator = data.dataset.make_initializable_iterator()
next_element = iterator.get_next()
sess.run(iterator.initializer)
for idx in range(data.num_batches):
try:
dataset = sess.run(next_element)
except tf.errors.OutOfRangeError:
break
feed_dict = feed.build_feed_dict(model, dataset,
data.max_sentence_length, True)
if 'bert' in model.config.emb_class:
# compute bert embedding at runtime
bert_embeddings = sess.run([model.bert_embeddings_subgraph],
feed_dict=feed_dict,
options=runopts)
if idx == 0:
tf.logging.debug('# bert_token_ids')
t = dataset['bert_token_ids'][:1]
tf.logging.debug(' '.join([str(x) for x in np.shape(t)]))
tf.logging.debug(' '.join([str(x) for x in t]))
tf.logging.debug('# bert_token_masks')
t = dataset['bert_token_masks'][:1]
tf.logging.debug(' '.join([str(x) for x in np.shape(t)]))
tf.logging.debug(' '.join([str(x) for x in t]))
tf.logging.debug('# bert_embedding')
t = bert_embeddings[0][:1]
tf.logging.debug(' '.join([str(x) for x in np.shape(t)]))
tf.logging.debug(' '.join([str(x) for x in t]))
tf.logging.debug('# bert_wordidx2tokenidx')
t = dataset['bert_wordidx2tokenidx'][:1]
tf.logging.debug(' '.join([str(x) for x in np.shape(t)]))
tf.logging.debug(' '.join([str(x) for x in t]))
# update feed_dict
feed.update_feed_dict(model, feed_dict, bert_embeddings,
dataset['bert_wordidx2tokenidx'])
step, summaries, _, loss, accuracy, f1, learning_rate = \
sess.run([model.global_step, summary_op, model.train_op, \
model.loss, model.accuracy, model.f1, \
model.learning_rate], feed_dict=feed_dict, options=runopts)
else:
step, summaries, _, loss, accuracy, f1, learning_rate = \
sess.run([model.global_step, summary_op, model.train_op, \
model.loss, model.accuracy, model.f1, \
model.learning_rate], feed_dict=feed_dict, options=runopts)
summary_writer.add_summary(summaries, step)
prog.update(idx + 1,
[('step', step), ('train loss', loss),
('train accuracy', accuracy), ('train f1', f1),
('lr(invalid if use_bert_optimization)', learning_rate)])
duration_time = time.time() - start_time
out = '\nduration_time : ' + str(duration_time) + ' sec for this epoch'
tf.logging.debug(out)
def run_epoch(self, sess, merged, data):
prog = Progbar(target=1 + int(data.data_size / TRAIN_BATCH_SIZE),
file_given=self.log)
losses = list()
i = 0
batch = data.get_selected_passage_batch()
while batch is not None:
q_batch = batch['question']
p_batch = batch['passage']
a_batch = batch['answer']
s_t_batch = batch['start_token']
dropout = batch['dropout']
loss = self.train_on_batch(sess, merged, q_batch, p_batch,
s_t_batch, dropout, a_batch)
tf.summary.scalar('Loss per Batch', loss)
losses.append(loss)
prog.update(i + 1, [("train loss", loss)])
batch = data.get_selected_passage_batch()
i += 1
return losses
def run_epoch(self, sess, merged, data):
prog = Progbar(target=1 + int(data.data_size / TRAIN_BATCH_SIZE), file_given=self.log)
losses = list()
i = 0
batch = data.get_selected_passage_batch()
while batch is not None:
q_batch = batch['question']
p_batch = batch['passage']
a_batch = batch['answer']
s_t_batch = batch['start_token']
dropout = batch['dropout']
self._temp_test_answer_indices = a_batch
loss = self.train_on_batch(sess, merged, q_batch, p_batch, s_t_batch, dropout, a_batch)
tf.summary.scalar('Loss per Batch', loss)
losses.append(loss)
prog.update(i + 1, [("train loss", loss)])
batch = data.get_selected_passage_batch()
if i % 1200 == 0 and i > 0:
self.log.write('\nNow saving file...')
saver.save(sess, SAVE_MODEL_DIR)
self.log.write('\nSaved...')
i += 1
return losses
def predict(self, sess, saver, data):
self.predicting = True
prog = Progbar(target=1 + int(data.data_size / TRAIN_BATCH_SIZE), file_given=self.log)
preds = list()
i = 0
data.reset_iter()
batch = data.get_selected_passage_batch(predicting=True)
while batch is not None:
q_batch = batch['question']
p_batch = batch['passage']
s_t_batch = batch['start_token']
a_batch = np.zeros((q_batch.shape[0], OUTPUT_MAX_LENGTH), dtype=np.int32)
dropout = batch['dropout']
prediction = self.predict_on_batch(sess, q_batch, p_batch, s_t_batch, dropout, a_batch)
preds.append(prediction)
prog.update(i + 1, [("Predictions going...", 1)])
batch = data.get_selected_passage_batch(predicting=True)
i += 1
return preds
def main(filenames, device='/gpu:2'):
print("[Info] Loading Model")
sess = tf.Session()
K.set_session(sess)
model = load_model(os.path.join(MODEL_DIR, 'from_tony.h5'))
to_tf = tf.placeholder(tf.float32, shape=(None, 16, 112, 112, 3))
output_tensor = model(to_tf)
mean_cube = np.load(os.path.join(MODEL_DIR, 'train01_16_128_171_mean.npy'))
mean_cube = np.transpose(mean_cube, (1, 2, 3, 0))
to_net = np.empty((16, 112, 112, 3))
for filename in filenames:
print("[Info] Loading videos")
vid = np.load(os.path.join(OUTDIR, 'preprocessed',
filename)).astype(np.float32)
temp = np.zeros((16,) + vid.shape[1:])
end_frame = vid.shape[0]
output = np.empty((vid.shape[0], 8192))
status = Progbar(target=end_frame, text=filename.rsplit('.', 1)[0])
for frm_ind, frame in enumerate(vid):
if frm_ind < 8:
temp[(8 - frm_ind):] = vid[:frm_ind + 8]
temp[:(8 - frm_ind)] = 0
elif end_frame < frm_ind + 8:
temp[:(16 + (end_frame - (frm_ind + 8)))] = vid[frm_ind - 8:]
else:
temp[:, :, :, :] = vid[(frm_ind - 8):(frm_ind + 8), :, :, :]
temp -= mean_cube
to_net[:, :, :, :] = temp[:, 8:120, 30:142, :]
output[frm_ind] = sess.run(output_tensor,
feed_dict={to_tf: np.array([to_net])})
# with tf.device(device):
# output[frm_ind] = model.predict_on_batch(np.array([to_net]))
if frm_ind % 50:
status.update(frm_ind)
np.save(os.path.join(OUTDIR, 'extracted', filename), output)
def evaluate(model, config, val_loader):
model.eval()
opt = config['opt']
pad_label_id = config['pad_label_id']
eval_loss = 0.
criterion = nn.CrossEntropyLoss(ignore_index=pad_label_id).to(opt.device)
n_batches = len(val_loader)
prog = Progbar(target=n_batches)
preds = None
ys = None
with torch.no_grad():
for i, (x, y) in enumerate(val_loader):
x = to_device(x, opt.device)
y = to_device(y, opt.device)
if opt.use_crf:
logits, prediction = model(x)
mask = torch.sign(torch.abs(x[0])).to(torch.uint8).to(
opt.device)
log_likelihood = model.crf(logits,
y,
mask=mask,
reduction='mean')
loss = -1 * log_likelihood
else:
logits = model(x)
loss = criterion(logits.view(-1, model.label_size), y.view(-1))
if preds is None:
if opt.use_crf: preds = to_numpy(prediction)
else: preds = to_numpy(logits)
ys = to_numpy(y)
else:
if opt.use_crf:
preds = np.append(preds, to_numpy(prediction), axis=0)
else:
preds = np.append(preds, to_numpy(logits), axis=0)
ys = np.append(ys, to_numpy(y), axis=0)
eval_loss += loss.item()
prog.update(i + 1, [('eval curr loss', loss.item())])
eval_loss = eval_loss / n_batches
if not opt.use_crf: preds = np.argmax(preds, axis=2)
# compute measure using seqeval
labels = model.labels
ys_lbs = [[] for _ in range(ys.shape[0])]
preds_lbs = [[] for _ in range(ys.shape[0])]
for i in range(ys.shape[0]): # foreach sentence
for j in range(ys.shape[1]): # foreach token
if ys[i][j] != pad_label_id:
ys_lbs[i].append(labels[ys[i][j]])
preds_lbs[i].append(labels[preds[i][j]])
ret = {
"loss": eval_loss,
"precision": precision_score(ys_lbs, preds_lbs),
"recall": recall_score(ys_lbs, preds_lbs),
"f1": f1_score(ys_lbs, preds_lbs),
"report": classification_report(ys_lbs, preds_lbs, digits=4),
}
print(ret['report'])
return ret
def get_data(label_dict, path):
lens = [arr.values.shape[0] for arr in label_dict.values()]
rtn_arr = np.empty((sum(lens), 128, 171, 3), dtype=np.float32)
status = Progbar(len(lens))
[(
fill_arr(ind, rtn_arr,
np.load(pthjn(path,
arr.rsplit('.', 1)[0] + '.npy')), lens,
lens[ind]),
status.update(ind, text=arr.rsplit('.', 1)[0]),
) for ind, arr in enumerate(label_dict.keys())]
return rtn_arr
def predict(self, sess, data):
self.testing = False
prog = Progbar(target=1 + int(data.data_size / TRAIN_BATCH_SIZE), file_given=self.log)
preds = list()
i = 0
batch = data.get_batch(predicting=True)
while batch is not None:
q_batch = batch['question']
p_batch = batch['passage']
dropout = batch['dropout']
prediction = self.predict_on_batch(sess, q_batch, p_batch, dropout)
preds.append(prediction)
prog.update(i + 1, [("Predictions going...", 1)])
batch = data.get_batch(predicting=True)
i += 1
return preds
def run_epoch(self, sess, data):
prog = Progbar(target=1 + int(data.data_size / TRAIN_BATCH_SIZE),
file_given=self.log)
losses = list()
i = 0
batch = data.get_batch()
while batch is not None:
q_batch = batch[0]
p_batch = batch[1]
a_batch = batch[2]
s_t_batch = batch[3]
loss = self.train_on_batch(sess, q_batch, p_batch, s_t_batch,
a_batch)
losses.append(loss)
prog.update(i + 1, [("train loss", loss)])
batch = data.get_batch()
i += 1
return losses
def predict(self, sess, saver, data):
prog = Progbar(target=1 + int(data.data_size / TRAIN_BATCH_SIZE),
file_given=self.log)
preds = list()
i = 0
batch = data.get_batch(batch_size=TRAIN_BATCH_SIZE)
while batch is not None:
q_batch = batch[0]
p_batch = batch[1]
s_t_batch = batch[3]
prediction = self.predict_on_batch(sess, q_batch, p_batch,
s_t_batch)
preds.append(prediction)
prog.update(i + 1, [("Predictions going...", 1)])
batch = data.get_batch(batch_size=TRAIN_BATCH_SIZE)
i += 1
return preds
def run_epoch(self, sess, data):
prog = Progbar(target=1 + int(data.data_size / TRAIN_BATCH_SIZE), file_given=self.log)
losses = list()
i = 0
batch = data.get_batch()
while batch is not None:
q_batch = batch[0]
p_batch = batch[1]
a_batch = batch[2]
s_t_batch = batch[3]
loss = self.train_on_batch(sess, q_batch, p_batch, s_t_batch, a_batch)
losses.append(loss)
prog.update(i + 1, [("train loss", loss)])
batch = data.get_batch()
if i % 1200 == 0 and i > 0:
self.log.write('\nNow saving file...')
saver.save(sess, SAVE_MODEL_DIR)
self.log.write('\nSaved...')
i += 1
return losses
def _work(self, cls):
frame_cnt = 0
self.long_enough = self.lens_df.loc[cls].apply(
lambda lst: map(lambda lgn: lgn > self.min_len, lst))
status = Progbar(self.min_frame_cnt, text=cls)
self.chk_long_enough()
while frame_cnt < self.min_frame_cnt:
self.chk_long_enough()
if len(self.long_enough.index) == 0:
status.update(frame_cnt, force=True)
print("\n")
return
smp_pick = choice(list(enumerate(self.long_enough.values)))
run_pick = choice(list(enumerate(smp_pick[-1])))
while not run_pick[-1]:
run_pick = choice(list(enumerate(smp_pick[-1])))
src = self.long_enough.index.values[smp_pick[0]]
self.run.append((src,
self.runs_df.loc[cls, src]
[run_pick[0]],))
frame_cnt += self.lens_df.loc[cls, src][run_pick[0]]
status.update(frame_cnt)
self.long_enough.iloc[smp_pick[0]][run_pick[0]] = False
self.chk_long_enough()
def train_epoch(model, config, train_loader, val_loader, epoch_i):
opt = config['opt']
optimizer = config['optimizer']
scheduler = config['scheduler']
writer = config['writer']
scaler = config['scaler']
pad_label_id = config['pad_label_id']
criterion = nn.CrossEntropyLoss(ignore_index=pad_label_id).to(opt.device)
n_batches = len(train_loader)
prog = Progbar(target=n_batches)
# train one epoch
model.train()
train_loss = 0.
st_time = time.time()
optimizer.zero_grad()
for local_step, (x,y) in enumerate(train_loader):
global_step = (len(train_loader) * epoch_i) + local_step
x = to_device(x, opt.device)
y = to_device(y, opt.device)
if opt.use_crf:
with autocast(enabled=opt.use_amp):
if opt.use_profiler:
with profiler.profile(profile_memory=True, record_shapes=True) as prof:
logits, prediction = model(x)
print(prof.key_averages().table(sort_by="self_cpu_memory_usage", row_limit=10))
else:
logits, prediction = model(x)
mask = torch.sign(torch.abs(x[0])).to(torch.uint8).to(opt.device)
log_likelihood = model.crf(logits, y, mask=mask, reduction='mean')
loss = -1 * log_likelihood
if opt.gradient_accumulation_steps > 1:
loss = loss / opt.gradient_accumulation_steps
else:
with autocast(enabled=opt.use_amp):
if opt.use_profiler:
with profiler.profile(profile_memory=True, record_shapes=True) as prof:
logits = model(x)
print(prof.key_averages().table(sort_by="self_cpu_memory_usage", row_limit=10))
else:
logits = model(x)
# reshape for computing loss
logits_view = logits.view(-1, model.label_size)
y_view = y.view(-1)
loss = criterion(logits_view, y_view)
if opt.gradient_accumulation_steps > 1:
loss = loss / opt.gradient_accumulation_steps
# back-propagation - begin
scaler.scale(loss).backward()
if (local_step + 1) % opt.gradient_accumulation_steps == 0:
scaler.unscale_(optimizer)
torch.nn.utils.clip_grad_norm_(model.parameters(), opt.max_grad_norm)
scaler.step(optimizer)
scaler.update()
optimizer.zero_grad()
if opt.use_transformers_optimizer: scheduler.step()
# back-propagation - end
train_loss += loss.item()
if writer: writer.add_scalar('Loss/train', loss.item(), global_step)
curr_lr = scheduler.get_last_lr()[0] if scheduler else optimizer.param_groups[0]['lr']
prog.update(local_step+1,
[('global step', global_step),
('train curr loss', loss.item()),
('lr', curr_lr)])
train_loss = train_loss / n_batches
# evaluate
eval_ret = evaluate(model, config, val_loader)
eval_loss = eval_ret['loss']
eval_f1 = eval_ret['f1']
curr_time = time.time()
elapsed_time = (curr_time - st_time) / 60
st_time = curr_time
curr_lr = scheduler.get_last_lr()[0] if scheduler else optimizer.param_groups[0]['lr']
logger.info('{:3d} epoch | {:5d}/{:5d} | train loss : {:10.6f}, valid loss {:10.6f}, valid f1 {:.4f}| lr :{:7.6f} | {:5.2f} min elapsed'.\
format(epoch_i, local_step+1, len(train_loader), train_loss, eval_loss, eval_f1, curr_lr, elapsed_time))
if writer:
writer.add_scalar('Loss/valid', eval_loss, global_step)
writer.add_scalar('F1/valid', eval_f1, global_step)
writer.add_scalar('LearningRate/train', curr_lr, global_step)
return eval_loss, eval_f1
def dev_step(model, data, summary_writer, epoch):
"""Evaluate dev data
"""
sess = model.sess
runopts = tf.RunOptions(report_tensor_allocations_upon_oom=True)
sum_loss = 0.0
sum_accuracy = 0.0
sum_f1 = 0.0
sum_output_indices = None
sum_logits_indices = None
sum_sentence_lengths = None
trans_params = None
global_step = 0
prog = Progbar(target=data.num_batches)
iterator = data.dataset.make_initializable_iterator()
next_element = iterator.get_next()
sess.run(iterator.initializer)
# evaluate on dev data sliced by batch_size to prevent OOM(Out Of Memory).
for idx in range(data.num_batches):
try:
dataset = sess.run(next_element)
except tf.errors.OutOfRangeError:
break
feed_dict = feed.build_feed_dict(model, dataset,
data.max_sentence_length, False)
if 'bert' in model.config.emb_class:
# compute bert embedding at runtime
bert_embeddings = sess.run([model.bert_embeddings_subgraph],
feed_dict=feed_dict,
options=runopts)
# update feed_dict
feed.update_feed_dict(model, feed_dict, bert_embeddings,
dataset['bert_wordidx2tokenidx'])
global_step, logits_indices, sentence_lengths, loss, accuracy, f1 = \
sess.run([model.global_step, model.logits_indices, model.sentence_lengths, \
model.loss, model.accuracy, model.f1], feed_dict=feed_dict)
prog.update(idx + 1, [('dev loss', loss), ('dev accuracy', accuracy),
('dev f1', f1)])
sum_loss += loss
sum_accuracy += accuracy
sum_f1 += f1
sum_output_indices = np_concat(sum_output_indices,
np.argmax(dataset['tags'], 2))
sum_logits_indices = np_concat(sum_logits_indices, logits_indices)
sum_sentence_lengths = np_concat(sum_sentence_lengths,
sentence_lengths)
idx += 1
avg_loss = sum_loss / data.num_batches
avg_accuracy = sum_accuracy / data.num_batches
avg_f1 = sum_f1 / data.num_batches
tag_preds = model.config.logits_indices_to_tags_seq(
sum_logits_indices, sum_sentence_lengths)
tag_corrects = model.config.logits_indices_to_tags_seq(
sum_output_indices, sum_sentence_lengths)
tf.logging.debug('\n[epoch %s/%s] dev precision, recall, f1(token): ' %
(epoch, model.config.epoch))
token_f1, l_token_prec, l_token_rec, l_token_f1 = TokenEval.compute_f1(
model.config.class_size, sum_logits_indices, sum_output_indices,
sum_sentence_lengths)
tf.logging.debug('[' + ' '.join([str(x) for x in l_token_prec]) + ']')
tf.logging.debug('[' + ' '.join([str(x) for x in l_token_rec]) + ']')
tf.logging.debug('[' + ' '.join([str(x) for x in l_token_f1]) + ']')
chunk_prec, chunk_rec, chunk_f1 = ChunkEval.compute_f1(
tag_preds, tag_corrects)
tf.logging.debug('dev precision(chunk), recall(chunk), f1(chunk): %s, %s, %s' % \
(chunk_prec, chunk_rec, chunk_f1) + \
'(invalid for bert due to X tag)')
# create summaries manually.
summary_value = [
tf.Summary.Value(tag='loss', simple_value=avg_loss),
tf.Summary.Value(tag='accuracy', simple_value=avg_accuracy),
tf.Summary.Value(tag='f1', simple_value=avg_f1),
tf.Summary.Value(tag='token_f1', simple_value=token_f1),
tf.Summary.Value(tag='chunk_f1', simple_value=chunk_f1)
]
summaries = tf.Summary(value=summary_value)
summary_writer.add_summary(summaries, global_step)
return token_f1, chunk_f1, avg_f1
def dev_step(model, data, summary_writer, epoch):
"""Evaluate dev data
"""
def np_concat(sum_var, var):
if sum_var is not None:
sum_var = np.concatenate((sum_var, var), axis=0)
else:
sum_var = var
return sum_var
sess = model.sess
runopts = tf.RunOptions(report_tensor_allocations_upon_oom=True)
sum_loss = 0.0
sum_accuracy = 0.0
sum_f1 = 0.0
sum_output_indices = None
sum_logits_indices = None
sum_sentence_lengths = None
trans_params = None
global_step = 0
prog = Progbar(target=data.num_batches)
iterator = data.dataset.make_initializable_iterator()
next_element = iterator.get_next()
sess.run(iterator.initializer)
# evaluate on dev data sliced by batch_size to prevent OOM(Out Of Memory).
for idx in range(data.num_batches):
try:
dataset = sess.run(next_element)
except tf.errors.OutOfRangeError:
break
feed_dict = feed.build_feed_dict(model, dataset,
data.max_sentence_length, False)
if 'bert' in model.config.emb_class:
# compute bert embedding at runtime
bert_embeddings = sess.run([model.bert_embeddings_subgraph],
feed_dict=feed_dict,
options=runopts)
# update feed_dict
feed_dict[model.bert_embeddings] = feed.align_bert_embeddings(
config, bert_embeddings, dataset['bert_wordidx2tokenidx'], idx)
global_step, logits_indices, sentence_lengths, loss, accuracy, f1 = \
sess.run([model.global_step, model.logits_indices, model.sentence_lengths, \
model.loss, model.accuracy, model.f1], feed_dict=feed_dict)
prog.update(idx + 1, [('dev loss', loss), ('dev accuracy', accuracy),
('dev f1(tf_metrics)', f1)])
sum_loss += loss
sum_accuracy += accuracy
sum_f1 += f1
sum_output_indices = np_concat(sum_output_indices,
np.argmax(dataset['tags'], 2))
sum_logits_indices = np_concat(sum_logits_indices, logits_indices)
sum_sentence_lengths = np_concat(sum_sentence_lengths,
sentence_lengths)
idx += 1
avg_loss = sum_loss / data.num_batches
avg_accuracy = sum_accuracy / data.num_batches
avg_f1 = sum_f1 / data.num_batches
tag_preds = model.config.logits_indices_to_tags_seq(
sum_logits_indices, sum_sentence_lengths)
tag_corrects = model.config.logits_indices_to_tags_seq(
sum_output_indices, sum_sentence_lengths)
seqeval_prec = precision_score(tag_corrects, tag_preds)
seqeval_rec = recall_score(tag_corrects, tag_preds)
seqeval_f1 = f1_score(tag_corrects, tag_preds)
tf.logging.debug('\n[epoch %s/%s] dev precision(seqeval), recall(seqeval), f1(seqeval): %s, %s, %s' % \
(epoch, model.config.epoch, seqeval_prec, seqeval_rec, seqeval_f1))
# create summaries manually.
summary_value = [
tf.Summary.Value(tag='loss', simple_value=avg_loss),
tf.Summary.Value(tag='accuracy', simple_value=avg_accuracy),
tf.Summary.Value(tag='f1(tf_metrics)', simple_value=avg_f1),
tf.Summary.Value(tag='f1(seqeval)', simple_value=seqeval_f1)
]
summaries = tf.Summary(value=summary_value)
summary_writer.add_summary(summaries, global_step)
return seqeval_f1, avg_f1
def dev_step(sess, model, config, data, summary_writer, epoch):
idx = 0
nbatches = (len(data.sentence_tags) + config.dev_batch_size -
1) // config.dev_batch_size
prog = Progbar(target=nbatches)
sum_loss = 0.0
sum_accuracy = 0.0
sum_logits = None
sum_sentence_lengths = None
trans_params = None
global_step = 0
# evaluate on dev data sliced by dev_batch_size to prevent OOM
for ptr in range(0, len(data.sentence_tags), config.dev_batch_size):
config.is_training = False
feed_dict = {
model.input_data_pos_ids:
data.sentence_pos_ids[ptr:ptr + config.dev_batch_size],
model.output_data:
data.sentence_tags[ptr:ptr + config.dev_batch_size],
model.is_train:
config.is_training,
model.sentence_length:
data.max_sentence_length
}
feed_dict[model.input_data_word_ids] = data.sentence_word_ids[
ptr:ptr + config.dev_batch_size]
feed_dict[model.input_data_wordchr_ids] = data.sentence_wordchr_ids[
ptr:ptr + config.dev_batch_size]
if config.emb_class == 'elmo':
feed_dict[
model.
elmo_input_data_wordchr_ids] = data.sentence_elmo_wordchr_ids[
ptr:ptr + config.dev_batch_size]
if config.emb_class == 'bert':
feed_dict[
model.
bert_input_data_token_ids] = data.sentence_bert_token_ids[
ptr:ptr + config.batch_size]
feed_dict[
model.
bert_input_data_token_masks] = data.sentence_bert_token_masks[
ptr:ptr + config.batch_size]
feed_dict[
model.
bert_input_data_segment_ids] = data.sentence_bert_segment_ids[
ptr:ptr + config.batch_size]
global_step, logits, trans_params, sentence_lengths, loss, accuracy = \
sess.run([model.global_step, model.logits, model.trans_params, model.sentence_lengths, \
model.loss, model.accuracy], feed_dict=feed_dict)
prog.update(idx + 1, [('dev loss', loss), ('dev accuracy', accuracy)])
sum_loss += loss
sum_accuracy += accuracy
sum_logits = np_concat(sum_logits, logits)
sum_sentence_lengths = np_concat(sum_sentence_lengths,
sentence_lengths)
idx += 1
sum_loss = sum_loss / nbatches
sum_accuracy = sum_accuracy / nbatches
print('[epoch %s/%s] dev precision, recall, f1(token): ' %
(epoch, config.epoch))
token_f1 = TokenEval.compute_f1(config.class_size, sum_logits,
data.sentence_tags, sum_sentence_lengths)
if config.use_crf:
viterbi_sequences = viterbi_decode(sum_logits, trans_params,
sum_sentence_lengths)
tag_preds = data.logits_indices_to_tags_seq(viterbi_sequences,
sum_sentence_lengths)
else:
sum_logits_indices = np.argmax(sum_logits, 2)
tag_preds = data.logits_indices_to_tags_seq(sum_logits_indices,
sum_sentence_lengths)
sum_output_data_indices = np.argmax(data.sentence_tags, 2)
tag_corrects = data.logits_indices_to_tags_seq(sum_output_data_indices,
sum_sentence_lengths)
prec, rec, f1 = ChunkEval.compute_f1(tag_preds, tag_corrects)
print('dev precision, recall, f1(chunk): ', prec, rec, f1,
', this is no meaningful for emb_class=bert')
chunk_f1 = f1
m = chunk_f1
# create summaries manually
summary_value = [
tf.Summary.Value(tag='loss_1', simple_value=sum_loss),
tf.Summary.Value(tag='accuracy_1', simple_value=sum_accuracy),
tf.Summary.Value(tag='token_f1', simple_value=token_f1),
tf.Summary.Value(tag='chunk_f1', simple_value=chunk_f1)
]
summaries = tf.Summary(value=summary_value)
summary_writer.add_summary(summaries, global_step)
m = token_f1
return m
def train_step(sess, model, config, data, summary_op, summary_writer):
start_time = time.time()
runopts = tf.RunOptions(report_tensor_allocations_upon_oom=True)
idx = 0
nbatches = (len(data.sentence_tags) + config.batch_size -
1) // config.batch_size
prog = Progbar(target=nbatches)
for ptr in range(0, len(data.sentence_tags), config.batch_size):
config.is_training = True
feed_dict = {
model.input_data_pos_ids:
data.sentence_pos_ids[ptr:ptr + config.batch_size],
model.output_data:
data.sentence_tags[ptr:ptr + config.batch_size],
model.is_train:
config.is_training,
model.sentence_length:
data.max_sentence_length
}
feed_dict[model.input_data_word_ids] = data.sentence_word_ids[
ptr:ptr + config.batch_size]
feed_dict[model.input_data_wordchr_ids] = data.sentence_wordchr_ids[
ptr:ptr + config.batch_size]
if config.emb_class == 'elmo':
feed_dict[
model.
elmo_input_data_wordchr_ids] = data.sentence_elmo_wordchr_ids[
ptr:ptr + config.batch_size]
if config.emb_class == 'bert':
feed_dict[
model.
bert_input_data_token_ids] = data.sentence_bert_token_ids[
ptr:ptr + config.batch_size]
feed_dict[
model.
bert_input_data_token_masks] = data.sentence_bert_token_masks[
ptr:ptr + config.batch_size]
feed_dict[
model.
bert_input_data_segment_ids] = data.sentence_bert_segment_ids[
ptr:ptr + config.batch_size]
step, summaries, _, loss, accuracy, learning_rate, bert_embeddings = \
sess.run([model.global_step, summary_op, model.train_op, \
model.loss, model.accuracy, model.learning_rate, model.bert_embeddings], feed_dict=feed_dict, options=runopts)
if config.emb_class == 'bert' and idx == 0:
print("# bert_token_ids")
t = data.sentence_bert_token_ids[:3]
print(np.shape(t))
print(t)
print("# bert_token_masks")
t = data.sentence_bert_token_masks[:3]
print(np.shape(t))
print(t)
print("# bert_embedding")
t = bert_embeddings[:3]
print(np.shape(t))
print(t)
summary_writer.add_summary(summaries, step)
prog.update(idx + 1, [('step', step), ('train loss', loss),
('train accuracy', accuracy),
('lr', learning_rate)])
idx += 1
duration_time = time.time() - start_time
out = 'duration_time : ' + str(duration_time) + ' sec for this epoch'
sys.stderr.write(out + '\n')