def train_step(model, data, summary_op, summary_writer): """Train one epoch """ start_time = time.time() sess = model.sess runopts = tf.RunOptions(report_tensor_allocations_upon_oom=True) prog = Progbar(target=data.num_batches) iterator = data.dataset.make_initializable_iterator() next_element = iterator.get_next() sess.run(iterator.initializer) for idx in range(data.num_batches): try: dataset = sess.run(next_element) except tf.errors.OutOfRangeError: break feed_dict = feed.build_feed_dict(model, dataset, data.max_sentence_length, True) if 'bert' in model.config.emb_class: # compute bert embedding at runtime bert_embeddings = sess.run([model.bert_embeddings_subgraph], feed_dict=feed_dict, options=runopts) if idx == 0: tf.logging.debug('# bert_token_ids') t = dataset['bert_token_ids'][:1] tf.logging.debug(' '.join([str(x) for x in np.shape(t)])) tf.logging.debug(' '.join([str(x) for x in t])) tf.logging.debug('# bert_token_masks') t = dataset['bert_token_masks'][:1] tf.logging.debug(' '.join([str(x) for x in np.shape(t)])) tf.logging.debug(' '.join([str(x) for x in t])) tf.logging.debug('# bert_wordidx2tokenidx') t = dataset['bert_wordidx2tokenidx'][:1] tf.logging.debug(' '.join([str(x) for x in np.shape(t)])) tf.logging.debug(' '.join([str(x) for x in t])) # update feed_dict feed_dict[model.bert_embeddings] = feed.align_bert_embeddings( config, bert_embeddings, dataset['bert_wordidx2tokenidx'], idx) step, summaries, _, loss, accuracy, f1, learning_rate = \ sess.run([model.global_step, summary_op, model.train_op, \ model.loss, model.accuracy, model.f1, \ model.learning_rate], feed_dict=feed_dict, options=runopts) else: step, summaries, _, loss, accuracy, f1, learning_rate = \ sess.run([model.global_step, summary_op, model.train_op, \ model.loss, model.accuracy, model.f1, \ model.learning_rate], feed_dict=feed_dict, options=runopts) summary_writer.add_summary(summaries, step) prog.update(idx + 1, [('step', step), ('train loss', loss), ('train accuracy', accuracy), ('train f1', f1), ('lr(invalid if use_bert_optimization)', learning_rate)]) duration_time = time.time() - start_time out = '\nduration_time : ' + str(duration_time) + ' sec for this epoch' tf.logging.debug(out)
def analyze(graph, sess, query, config, nlp): """Analyze query by nlp, etagger """ bucket = build_bucket(nlp, query) inp, feed_dict = feed.build_input_feed_dict_with_graph( graph, config, bucket, Input) ## mapping output/input tensors for bert if 'bert' in config.emb_class: t_bert_embeddings_subgraph = graph.get_tensor_by_name( 'prefix/bert_embeddings_subgraph:0') p_bert_embeddings = graph.get_tensor_by_name( 'prefix/bert_embeddings:0') ## mapping output tensors t_logits_indices = graph.get_tensor_by_name('prefix/logits_indices:0') t_sentence_lengths = graph.get_tensor_by_name('prefix/sentence_lengths:0') ## analyze if 'bert' in config.emb_class: # compute bert embedding at runtime bert_embeddings = sess.run([t_bert_embeddings_subgraph], feed_dict=feed_dict) # update feed_dict feed_dict[p_bert_embeddings] = feed.align_bert_embeddings( config, bert_embeddings, inp.example['bert_wordidx2tokenidx'], -1) logits_indices, sentence_lengths = sess.run( [t_logits_indices, t_sentence_lengths], feed_dict=feed_dict) tags = config.logit_indices_to_tags(logits_indices[0], sentence_lengths[0]) ## build output out = [] for i in range(len(bucket)): tmp = bucket[i] + ' ' + tags[i] tl = tmp.split() entry = {} entry['id'] = i entry['word'] = tl[0] entry['pos'] = tl[1] entry['chk'] = tl[2] entry['tag'] = tl[3] entry['predict'] = tl[4] out.append(entry) return out
def inference_bucket(config): """Inference for bucket. """ # create model and compile model = Model(config) model.compile() sess = model.sess # restore model saver = tf.train.Saver() saver.restore(sess, config.restore) sys.stderr.write('model restored' + '\n') ''' print(tf.global_variables()) print(tf.trainable_variables()) ''' num_buckets = 0 total_duration_time = 0.0 bucket = [] while 1: try: line = sys.stdin.readline() except KeyboardInterrupt: break if not line: break line = line.strip() if not line and len(bucket) >= 1: start_time = time.time() inp, feed_dict = feed.build_input_feed_dict(model, bucket, Input) if 'bert' in config.emb_class: # compute bert embedding at runtime bert_embeddings = sess.run([model.bert_embeddings_subgraph], feed_dict=feed_dict) # update feed_dict feed_dict[model.bert_embeddings] = feed.align_bert_embeddings( config, bert_embeddings, inp.example['bert_wordidx2tokenidx'], -1) logits_indices, sentence_lengths = sess.run( [model.logits_indices, model.sentence_lengths], feed_dict=feed_dict) tags = config.logit_indices_to_tags(logits_indices[0], sentence_lengths[0]) for i in range(len(bucket)): out = bucket[i] + ' ' + tags[i] sys.stdout.write(out + '\n') sys.stdout.write('\n') bucket = [] duration_time = time.time() - start_time out = 'duration_time : ' + str(duration_time) + ' sec' tf.logging.info(out) num_buckets += 1 if num_buckets != 1: # first one may takes longer time, so ignore in computing duration. total_duration_time += duration_time if line: bucket.append(line) if len(bucket) != 0: start_time = time.time() inp, feed_dict = feed.build_input_feed_dict(model, bucket, Input) if 'bert' in config.emb_class: # compute bert embedding at runtime bert_embeddings = sess.run([model.bert_embeddings_subgraph], feed_dict=feed_dict) # update feed_dict feed_dict[model.bert_embeddings] = feed.align_bert_embeddings( config, bert_embeddings, inp.example['bert_wordidx2tokenidx'], -1) logits_indices, sentence_lengths = sess.run( [model.logits_indices, model.sentence_lengths], feed_dict=feed_dict) tags = config.logit_indices_to_tags(logits_indices[0], sentence_lengths[0]) for i in range(len(bucket)): out = bucket[i] + ' ' + tags[i] sys.stdout.write(out + '\n') sys.stdout.write('\n') duration_time = time.time() - start_time out = 'duration_time : ' + str(duration_time) + ' sec' tf.logging.info(out) num_buckets += 1 total_duration_time += duration_time out = 'total_duration_time : ' + str(total_duration_time) + ' sec' + '\n' out += 'average processing time / bucket : ' + str( total_duration_time / (num_buckets - 1)) + ' sec' tf.logging.info(out) sess.close()
def inference_line(config): """Inference for raw string. """ def get_entity(doc, begin, end): for ent in doc.ents: # check included if ent.start_char <= begin and end <= ent.end_char: if ent.start_char == begin: return 'B-' + ent.label_ else: return 'I-' + ent.label_ return 'O' def build_bucket(nlp, line): bucket = [] doc = nlp(line) for token in doc: begin = token.idx end = begin + len(token.text) - 1 temp = [] ''' print(token.i, token.text, token.lemma_, token.pos_, token.tag_, token.dep_, token.shape_, token.is_alpha, token.is_stop, begin, end) ''' temp.append(token.text) temp.append(token.tag_) temp.append('O') # no chunking info entity = get_entity(doc, begin, end) temp.append(entity) # entity by spacy temp = ' '.join(temp) bucket.append(temp) return bucket import spacy nlp = spacy.load('en') # create model and compile model = Model(config) model.compile() sess = model.sess # restore model saver = tf.train.Saver() saver.restore(sess, config.restore) tf.logging.info('model restored' + '\n') while 1: try: line = sys.stdin.readline() except KeyboardInterrupt: break if not line: break line = line.strip() if not line: continue # create bucket try: bucket = build_bucket(nlp, line) except Exception as e: sys.stderr.write(str(e) + '\n') continue inp, feed_dict = feed.build_input_feed_dict(model, bucket) if 'bert' in config.emb_class: # compute bert embedding at runtime bert_embeddings = sess.run([model.bert_embeddings_subgraph], feed_dict=feed_dict) # update feed_dict feed_dict[model.bert_embeddings] = feed.align_bert_embeddings( config, bert_embeddings, inp.example['bert_wordidx2tokenidx'], -1) logits_indices, sentence_lengths = sess.run( [model.logits_indices, model.sentence_lengths], feed_dict=feed_dict) tags = config.logit_indices_to_tags(logits_indices[0], sentence_lengths[0]) for i in range(len(bucket)): out = bucket[i] + ' ' + tags[i] sys.stdout.write(out + '\n') sys.stdout.write('\n') sess.close()
def inference(config, frozen_pb_path): """Inference for bucket """ # load graph graph = load_frozen_graph(frozen_pb_path) for op in graph.get_operations(): sys.stderr.write(op.name + '\n') # create session with graph # if graph is optimized by tensorRT, then # from tensorflow.contrib import tensorrt as trt # gpu_ops = tf.GPUOptions(per_process_gpu_memory_fraction = 0.50) gpu_ops = tf.GPUOptions() ''' session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, gpu_options=gpu_ops) ''' session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, gpu_options=gpu_ops, inter_op_parallelism_threads=0, intra_op_parallelism_threads=0) sess = tf.Session(graph=graph, config=session_conf) # mapping output/input tensors for bert if 'bert' in config.emb_class: t_bert_embeddings_subgraph = graph.get_tensor_by_name( 'prefix/bert_embeddings_subgraph:0') p_bert_embeddings = graph.get_tensor_by_name( 'prefix/bert_embeddings:0') # mapping output tensors t_logits_indices = graph.get_tensor_by_name('prefix/logits_indices:0') t_sentence_lengths = graph.get_tensor_by_name('prefix/sentence_lengths:0') num_buckets = 0 total_duration_time = 0.0 bucket = [] while 1: try: line = sys.stdin.readline() except KeyboardInterrupt: break if not line: break line = line.strip() if not line and len(bucket) >= 1: start_time = time.time() inp, feed_dict = feed.build_input_feed_dict_with_graph( graph, config, bucket, Input) if 'bert' in config.emb_class: # compute bert embedding at runtime bert_embeddings = sess.run([t_bert_embeddings_subgraph], feed_dict=feed_dict) # update feed_dict feed_dict[p_bert_embeddings] = feed.align_bert_embeddings( config, bert_embeddings, inp.example['bert_wordidx2tokenidx'], -1) logits_indices, sentence_lengths = sess.run( [t_logits_indices, t_sentence_lengths], feed_dict=feed_dict) tags = config.logit_indices_to_tags(logits_indices[0], sentence_lengths[0]) for i in range(len(bucket)): out = bucket[i] + ' ' + tags[i] sys.stdout.write(out + '\n') sys.stdout.write('\n') bucket = [] duration_time = time.time() - start_time out = 'duration_time : ' + str(duration_time) + ' sec' sys.stderr.write(out + '\n') num_buckets += 1 if num_buckets != 1: # first one may takes longer time, so ignore in computing duration. total_duration_time += duration_time if line: bucket.append(line) if len(bucket) != 0: start_time = time.time() inp, feed_dict = feed.build_input_feed_dict_with_graph( graph, config, bucket, Input) if 'bert' in config.emb_class: # compute bert embedding at runtime bert_embeddings = sess.run([t_bert_embeddings_subgraph], feed_dict=feed_dict) # update feed_dict feed_dict[p_bert_embeddings] = feed.align_bert_embeddings( config, bert_embeddings, inp.example['bert_wordidx2tokenidx'], -1) logits_indices, sentence_lengths = sess.run( [t_logits_indices, t_sentence_lengths], feed_dict=feed_dict) tags = config.logit_indices_to_tags(logits_indices[0], sentence_lengths[0]) for i in range(len(bucket)): out = bucket[i] + ' ' + tags[i] sys.stdout.write(out + '\n') sys.stdout.write('\n') duration_time = time.time() - start_time out = 'duration_time : ' + str(duration_time) + ' sec' tf.logging.info(out) num_buckets += 1 total_duration_time += duration_time out = 'total_duration_time : ' + str(total_duration_time) + ' sec' + '\n' out += 'average processing time / bucket : ' + str( total_duration_time / (num_buckets - 1)) + ' sec' tf.logging.info(out) sess.close()
def dev_step(model, data, summary_writer, epoch): """Evaluate dev data """ def np_concat(sum_var, var): if sum_var is not None: sum_var = np.concatenate((sum_var, var), axis=0) else: sum_var = var return sum_var sess = model.sess runopts = tf.RunOptions(report_tensor_allocations_upon_oom=True) sum_loss = 0.0 sum_accuracy = 0.0 sum_f1 = 0.0 sum_output_indices = None sum_logits_indices = None sum_sentence_lengths = None trans_params = None global_step = 0 prog = Progbar(target=data.num_batches) iterator = data.dataset.make_initializable_iterator() next_element = iterator.get_next() sess.run(iterator.initializer) # evaluate on dev data sliced by batch_size to prevent OOM(Out Of Memory). for idx in range(data.num_batches): try: dataset = sess.run(next_element) except tf.errors.OutOfRangeError: break feed_dict = feed.build_feed_dict(model, dataset, data.max_sentence_length, False) if 'bert' in model.config.emb_class: # compute bert embedding at runtime bert_embeddings = sess.run([model.bert_embeddings_subgraph], feed_dict=feed_dict, options=runopts) # update feed_dict feed_dict[model.bert_embeddings] = feed.align_bert_embeddings( config, bert_embeddings, dataset['bert_wordidx2tokenidx'], idx) global_step, logits_indices, sentence_lengths, loss, accuracy, f1 = \ sess.run([model.global_step, model.logits_indices, model.sentence_lengths, \ model.loss, model.accuracy, model.f1], feed_dict=feed_dict) prog.update(idx + 1, [('dev loss', loss), ('dev accuracy', accuracy), ('dev f1(tf_metrics)', f1)]) sum_loss += loss sum_accuracy += accuracy sum_f1 += f1 sum_output_indices = np_concat(sum_output_indices, np.argmax(dataset['tags'], 2)) sum_logits_indices = np_concat(sum_logits_indices, logits_indices) sum_sentence_lengths = np_concat(sum_sentence_lengths, sentence_lengths) idx += 1 avg_loss = sum_loss / data.num_batches avg_accuracy = sum_accuracy / data.num_batches avg_f1 = sum_f1 / data.num_batches tag_preds = model.config.logits_indices_to_tags_seq( sum_logits_indices, sum_sentence_lengths) tag_corrects = model.config.logits_indices_to_tags_seq( sum_output_indices, sum_sentence_lengths) seqeval_prec = precision_score(tag_corrects, tag_preds) seqeval_rec = recall_score(tag_corrects, tag_preds) seqeval_f1 = f1_score(tag_corrects, tag_preds) tf.logging.debug('\n[epoch %s/%s] dev precision(seqeval), recall(seqeval), f1(seqeval): %s, %s, %s' % \ (epoch, model.config.epoch, seqeval_prec, seqeval_rec, seqeval_f1)) # create summaries manually. summary_value = [ tf.Summary.Value(tag='loss', simple_value=avg_loss), tf.Summary.Value(tag='accuracy', simple_value=avg_accuracy), tf.Summary.Value(tag='f1(tf_metrics)', simple_value=avg_f1), tf.Summary.Value(tag='f1(seqeval)', simple_value=seqeval_f1) ] summaries = tf.Summary(value=summary_value) summary_writer.add_summary(summaries, global_step) return seqeval_f1, avg_f1