def main(_): hvd.init() sess_config = tf.ConfigProto() sess_config.gpu_options.visible_device_list = str(hvd.local_rank()) graph = tf.Graph() with graph.as_default(): import json config = json.load(open(FLAGS.config_file, "r")) init_checkpoint = FLAGS.init_checkpoint config = Bunch(config) config.use_one_hot_embeddings = True config.scope = "bert" config.dropout_prob = 0.1 config.label_type = "single_label" if FLAGS.if_shard == "0": train_size = FLAGS.train_size epoch = int(FLAGS.epoch / hvd.size()) elif FLAGS.if_shard == "1": train_size = int(FLAGS.train_size/hvd.size()) epoch = FLAGS.epoch tokenizer = tokenization.FullTokenizer( vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.lower_case) classifier_data_api = classifier_processor.EvaluationProcessor() classifier_data_api.get_labels(FLAGS.label_id) train_examples = classifier_data_api.get_train_examples(FLAGS.train_file) write_to_tfrecords.convert_classifier_examples_to_features(train_examples, classifier_data_api.label2id, FLAGS.max_length, tokenizer, FLAGS.eval_data_file) init_lr = 2e-5 num_train_steps = int( train_size / FLAGS.batch_size * epoch) num_warmup_steps = int(num_train_steps * 0.1) num_storage_steps = int(train_size / FLAGS.batch_size) print(" model type {}".format(FLAGS.model_type)) print(num_train_steps, num_warmup_steps, "=============") opt_config = Bunch({"init_lr":init_lr/hvd.size(), "num_train_steps":num_train_steps, "num_warmup_steps":num_warmup_steps}) sess = tf.Session(config=sess_config) model_io_config = Bunch({"fix_lm":False}) model_io_fn = model_io.ModelIO(model_io_config) optimizer_fn = optimizer.Optimizer(opt_config) num_classes = FLAGS.num_classes model_eval_fn = bert_classifier.classifier_model_fn_builder(config, num_classes, init_checkpoint, reuse=False, load_pretrained=True, model_io_fn=model_io_fn, optimizer_fn=optimizer_fn, model_io_config=model_io_config, opt_config=opt_config) def metric_fn(features, logits, loss): print(logits.get_shape(), "===logits shape===") pred_label = tf.argmax(logits, axis=-1, output_type=tf.int32) prob = tf.nn.softmax(logits) accuracy = correct = tf.equal( tf.cast(pred_label, tf.int32), tf.cast(features["label_ids"], tf.int32) ) accuracy = tf.reduce_mean(tf.cast(correct, tf.float32)) return {"accuracy":accuracy, "loss":loss, "pred_label":pred_label, "label_ids":features["label_ids"], "prob":prob} name_to_features = { "input_ids": tf.FixedLenFeature([FLAGS.max_length], tf.int64), "input_mask": tf.FixedLenFeature([FLAGS.max_length], tf.int64), "segment_ids": tf.FixedLenFeature([FLAGS.max_length], tf.int64), "label_ids": tf.FixedLenFeature([], tf.int64), } def _decode_record(record, name_to_features): """Decodes a record to a TensorFlow example. """ example = tf.parse_single_example(record, name_to_features) # tf.Example only supports tf.int64, but the TPU only supports tf.int32. # So cast all int64 to int32. for name in list(example.keys()): t = example[name] if t.dtype == tf.int64: t = tf.to_int32(t) example[name] = t return example params = Bunch({}) params.epoch = FLAGS.epoch params.batch_size = FLAGS.batch_size eval_features = tf_data_utils.eval_input_fn(FLAGS.eval_data_file, _decode_record, name_to_features, params, if_shard=FLAGS.if_shard) [_, eval_loss, eval_per_example_loss, eval_logits] = model_eval_fn(eval_features, [], tf.estimator.ModeKeys.EVAL) result = metric_fn(eval_features, eval_logits, eval_loss) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess.run(init_op) sess.run(hvd.broadcast_global_variables(0)) print("===horovod rank==={}".format(hvd.rank())) def eval_fn(result): i = 0 total_accuracy = 0 label, label_id, prob = [], [], [] while True: try: eval_result = sess.run(result) total_accuracy += eval_result["accuracy"] label_id.extend(eval_result["label_ids"]) label.extend(eval_result["pred_label"]) prob.extend(eval_result["prob"]) i += 1 except tf.errors.OutOfRangeError: print("End of dataset") break macro_f1 = f1_score(label_id, label, average="macro") micro_f1 = f1_score(label_id, label, average="micro") macro_precision = precision_score(label_id, label, average="macro") micro_precision = precision_score(label_id, label, average="micro") macro_recall = recall_score(label_id, label, average="macro") micro_recall = recall_score(label_id, label, average="micro") accuracy = accuracy_score(label_id, label) print("test accuracy {} macro_f1 score {} micro_f1 {} accuracy {}".format(total_accuracy/ i, macro_f1, micro_f1, accuracy)) return total_accuracy/ i, label_id, label, prob import time import time start = time.time() acc, true_label, pred_label, prob = eval_fn(result) end = time.time() print("==total time {} numbers of devices {}".format(end - start, hvd.size())) if hvd.rank() == 0: import _pickle as pkl pkl.dump({"true_label":true_label, "pred_label":pred_label, "prob":prob}, open(FLAGS.model_output+"/predict.pkl", "wb"))
def model_fn(features, labels, mode): if model_io_config.fix_lm == True: scope = model_config.scope + "/task" else: scope = model_config.scope if mode == tf.estimator.ModeKeys.TRAIN: hidden_dropout_prob = model_config.hidden_dropout_prob attention_probs_dropout_prob = model_config.attention_probs_dropout_prob dropout_prob = model_config.dropout_prob else: hidden_dropout_prob = 0.0 attention_probs_dropout_prob = 0.0 dropout_prob = 0.0 label_ids = features["label_ids"] target = kargs["target"] [a_mask, a_repres, b_mask, b_repres] = bert_lstm_encoding(model_config, features, labels, mode, target, max_len, scope, dropout_prob, reuse=model_reuse) a_repres = lstm_model(model_config, a_repres, a_mask, dropout_prob, scope, model_reuse) b_repres = lstm_model(model_config, b_repres, b_mask, dropout_prob, scope, True) a_output, b_output = alignment(model_config, a_repres, b_repres, a_mask, b_mask, scope, reuse=model_reuse) repres_a = bert_multihead_pooling(model_config, a_output, a_mask, scope, dropout_prob, reuse=model_reuse) repres_b = bert_multihead_pooling(model_config, b_output, b_mask, scope, dropout_prob, reuse=True) pair_repres = tf.concat([ repres_a, repres_b, tf.abs(repres_a - repres_b), repres_b * repres_a ], axis=-1) print(pair_repres.get_shape(), "==repres shape==") with tf.variable_scope(scope, reuse=model_reuse): try: label_ratio_table = tf.get_variable( name="label_ratio", shape=[ num_labels, ], initializer=tf.constant(label_tensor), trainable=False) ratio_weight = tf.nn.embedding_lookup(label_ratio_table, label_ids) print("==applying class weight==") except: ratio_weight = None (loss, per_example_loss, logits) = classifier.classifier(model_config, pair_repres, num_labels, label_ids, dropout_prob, ratio_weight) if mode == tf.estimator.ModeKeys.TRAIN: pretrained_tvars = model_io_fn.get_params( model_config.scope, not_storage_params=not_storage_params) if load_pretrained: model_io_fn.load_pretrained( pretrained_tvars, init_checkpoint, exclude_scope=exclude_scope_dict["task"]) trainable_params = model_io_fn.get_params( scope, not_storage_params=not_storage_params) tvars = trainable_params storage_params = model_io_fn.get_params( model_config.scope, not_storage_params=not_storage_params) # for var in storage_params: # print(var.name, var.get_shape(), "==storage params==") # for var in tvars: # print(var.name, var.get_shape(), "==trainable params==") if mode == tf.estimator.ModeKeys.TRAIN: model_io_fn.print_params(tvars, string=", trainable params") update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): optimizer_fn = optimizer.Optimizer(opt_config) train_op = optimizer_fn.get_train_op( loss, tvars, opt_config.init_lr, opt_config.num_train_steps) return [train_op, loss, per_example_loss, logits] else: model_io_fn.print_params(tvars, string=", trainable params") return [loss, loss, per_example_loss, logits]
def main(_): hvd.init() sess_config = tf.ConfigProto() # sess_config.gpu_options.visible_device_list = str(hvd.local_rank()) sess_config.gpu_options.visible_device_list = \ '%d,%d' % (hvd.local_rank() * 2, hvd.local_rank() * 2 + 1) graph = tf.Graph() with graph.as_default(): import json config = json.load( open("/data/xuht/bert/chinese_L-12_H-768_A-12/bert_config.json", "r")) init_checkpoint = "/data/xuht/bert/chinese_L-12_H-768_A-12/bert_model.ckpt" config = Bunch(config) config.use_one_hot_embeddings = True config.scope = "bert" config.dropout_prob = 0.1 config.label_type = "single_label" config.loss = "focal_loss" # config.num_hidden_layers = # os.environ["CUDA_VISIBLE_DEVICES"] = "0" num_train = int(33056 / hvd.size()) batch_size = 32 valid_step = int(num_train / batch_size) epoch = 2 num_train_steps = int(num_train / (batch_size) * epoch) decay_train_steps = num_train_steps # decay_train_steps = int( # 33056 / batch_size * epoch) num_warmup_steps = int(num_train_steps * 0.01) sess = tf.Session(config=sess_config) opt_config = Bunch({ "init_lr": float(1e-5 / hvd.size()), "num_train_steps": decay_train_steps, "cycle": False, "num_warmup_steps": num_warmup_steps, "lr_decay": "polynomial_decay" }) model_io_config = Bunch({"fix_lm": False}) model_io_fn = model_io.ModelIO(model_io_config) optimizer_fn = optimizer.Optimizer(opt_config) num_calsses = 2 model_train_fn = bert_classifier.classifier_model_fn_builder( config, num_calsses, init_checkpoint, reuse=None, load_pretrained=True, model_io_fn=model_io_fn, optimizer_fn=optimizer_fn, model_io_config=model_io_config, opt_config=opt_config, gpu_id=0, gpu_nums=2) # model_eval_fn = bert_classifier.classifier_model_fn_builder(config, num_calsses, init_checkpoint, # reuse=True, # load_pretrained=True, # model_io_fn=model_io_fn, # optimizer_fn=optimizer_fn, # model_io_config=model_io_config, # opt_config=opt_config, # gpu_id=0, # gpu_nums=2) def metric_fn(features, logits, loss): print(logits.get_shape(), "===logits shape===") pred_label = tf.argmax(logits, axis=-1, output_type=tf.int32) prob = tf.nn.softmax(logits) accuracy = correct = tf.equal( tf.cast(pred_label, tf.int32), tf.cast(features["label_ids"], tf.int32)) accuracy = tf.reduce_mean(tf.cast(correct, tf.float32)) return { "accuracy": accuracy, "loss": loss, "pred_label": pred_label, "label_ids": features["label_ids"] } name_to_features = { "input_ids": tf.FixedLenFeature([128], tf.int64), "input_mask": tf.FixedLenFeature([128], tf.int64), "segment_ids": tf.FixedLenFeature([128], tf.int64), "label_ids": tf.FixedLenFeature([], tf.int64), } params = Bunch({}) params.epoch = epoch params.batch_size = 32 train_file = "/data/xuht/eventy_detection/event/model/train.tfrecords" train_file1 = "/data/xuht/eventy_detection/sentiment/model/sentiment_11_14/train.tfrecords" title_sentiment = "/data/xuht/eventy_detection/sentiment/model/test/train.tfrecords" sentiment = "/data/xuht/eventy_detection/sentiment/model/bert/train_11_15.tfrecords" jd_train = "/data/xuht/jd_comment/train.tfrecords" train_features = tf_data_utils.train_input_fn( jd_train, tf_data_utils._decode_record, name_to_features, params) test_file = [ "/data/xuht/eventy_detection/sentiment/model/sentiment_11_14/test.tfrecords" ] test_file1_1 = [ "/data/xuht/eventy_detection/sentiment/model/test/train.tfrecords", "/data/xuht/eventy_detection/sentiment/model/test/test.tfrecords" ] test_file2 = "/data/xuht/eventy_detection/event/model/test.tfrecords" title_test = "/data/xuht/eventy_detection/sentiment/model/test/test.tfrecords" jd_test = "/data/xuht/jd_comment/test.tfrecords" sentiment_test = "/data/xuht/eventy_detection/sentiment/model/bert/test_11_15.tfrecords" eval_features = tf_data_utils.eval_input_fn( jd_test, tf_data_utils._decode_record, name_to_features, params) [train_op, train_loss, train_per_example_loss, train_logits] = model_train_fn(train_features, [], tf.estimator.ModeKeys.TRAIN) # [_, eval_loss, eval_per_example_loss, eval_logits] = model_eval_fn(eval_features, [], tf.estimator.ModeKeys.EVAL) # result = metric_fn(eval_features, eval_logits, eval_loss) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess.run(init_op) sess.run(hvd.broadcast_global_variables(0)) model_io_fn.set_saver() print("===horovod rank==={}".format(hvd.rank())) def eval_fn(result): i = 0 total_accuracy = 0 label, label_id = [], [] while True: try: eval_result = sess.run(result) total_accuracy += eval_result["accuracy"] label_id.extend(eval_result["label_ids"]) label.extend(eval_result["pred_label"]) i += 1 except tf.errors.OutOfRangeError: print("End of dataset") break macro_f1 = f1_score(label_id, label, average="macro") micro_f1 = f1_score(label_id, label, average="micro") accuracy = accuracy_score(label_id, label) print("test accuracy {} macro_f1 score {} micro_f1 {} accuracy {}". format(total_accuracy / i, macro_f1, micro_f1, accuracy)) return total_accuracy / i, label_id, label def train_fn(op, loss): i = 0 total_loss = 0 cnt = 0 while True: try: [_, train_loss] = sess.run([op, loss]) i += 1 cnt += 1 total_loss += train_loss # print("==device id {} global step {}".format(hvd.rank(), step)) if np.mod(i, valid_step) == 0: print(total_loss / cnt) cnt = 0 total_loss = 0 except tf.errors.OutOfRangeError: print("End of dataset") break import time start = time.time() train_fn(train_op, train_loss) # acc, true_label, pred_label = eval_fn(result) end = time.time() print("==total time {} numbers of devices {}".format( end - start, hvd.size()))
def main(_): hvd.init() sess_config = tf.ConfigProto() sess_config.gpu_options.visible_device_list = str(hvd.local_rank()) graph = tf.Graph() with graph.as_default(): import json config = json.load(open(FLAGS.config_file, "r")) init_checkpoint = FLAGS.init_checkpoint config = Bunch(config) config.use_one_hot_embeddings = True config.scope = "bert" config.dropout_prob = 0.1 config.label_type = "single_label" if FLAGS.if_shard == "0": train_size = FLAGS.train_size epoch = int(FLAGS.epoch / hvd.size()) elif FLAGS.if_shard == "1": train_size = int(FLAGS.train_size / hvd.size()) epoch = FLAGS.epoch init_lr = 2e-5 label_dict = json.load(open(FLAGS.label_id)) num_train_steps = int(train_size / FLAGS.batch_size * epoch) num_warmup_steps = int(num_train_steps * 0.1) num_storage_steps = int(train_size / FLAGS.batch_size) print(" model type {}".format(FLAGS.model_type)) print(num_train_steps, num_warmup_steps, "=============") opt_config = Bunch({ "init_lr": init_lr / hvd.size(), "num_train_steps": num_train_steps, "num_warmup_steps": num_warmup_steps }) sess = tf.Session(config=sess_config) model_io_config = Bunch({"fix_lm": False}) model_io_fn = model_io.ModelIO(model_io_config) optimizer_fn = optimizer.Optimizer(opt_config) num_classes = FLAGS.num_classes model_train_fn = bert_classifier.classifier_model_fn_builder( config, num_classes, init_checkpoint, reuse=None, load_pretrained=True, model_io_fn=model_io_fn, optimizer_fn=optimizer_fn, model_io_config=model_io_config, opt_config=opt_config) model_eval_fn = bert_classifier.classifier_model_fn_builder( config, num_classes, init_checkpoint, reuse=True, load_pretrained=True, model_io_fn=model_io_fn, optimizer_fn=optimizer_fn, model_io_config=model_io_config, opt_config=opt_config) def metric_fn(features, logits, loss): print(logits.get_shape(), "===logits shape===") pred_label = tf.argmax(logits, axis=-1, output_type=tf.int32) prob = tf.nn.softmax(logits) accuracy = correct = tf.equal( tf.cast(pred_label, tf.int32), tf.cast(features["label_ids"], tf.int32)) accuracy = tf.reduce_mean(tf.cast(correct, tf.float32)) return { "accuracy": accuracy, "loss": loss, "pred_label": pred_label, "label_ids": features["label_ids"] } name_to_features = { "input_ids": tf.FixedLenFeature([FLAGS.max_length], tf.int64), "input_mask": tf.FixedLenFeature([FLAGS.max_length], tf.int64), "segment_ids": tf.FixedLenFeature([FLAGS.max_length], tf.int64), "label_ids": tf.FixedLenFeature([], tf.int64), } def _decode_record(record, name_to_features): """Decodes a record to a TensorFlow example. """ example = tf.parse_single_example(record, name_to_features) # tf.Example only supports tf.int64, but the TPU only supports tf.int32. # So cast all int64 to int32. for name in list(example.keys()): t = example[name] if t.dtype == tf.int64: t = tf.to_int32(t) example[name] = t return example params = Bunch({}) params.epoch = FLAGS.epoch params.batch_size = FLAGS.batch_size train_features = tf_data_utils.train_input_fn(FLAGS.train_file, _decode_record, name_to_features, params, if_shard=FLAGS.if_shard) eval_features = tf_data_utils.eval_input_fn(FLAGS.dev_file, _decode_record, name_to_features, params, if_shard=FLAGS.if_shard) [train_op, train_loss, train_per_example_loss, train_logits] = model_train_fn(train_features, [], tf.estimator.ModeKeys.TRAIN) train_dict = {"train_op": train_op, "train_loss": train_loss} [_, eval_loss, eval_per_example_loss, eval_logits] = model_eval_fn(eval_features, [], tf.estimator.ModeKeys.EVAL) eval_dict = metric_fn(eval_features, eval_logits, eval_loss) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess.run(init_op) sess.run(hvd.broadcast_global_variables(0)) model_io_fn.set_saver() print("===horovod rank==={}".format(hvd.rank())) def run_eval(steps): import _pickle as pkl # eval_features = tf_data_utils.eval_input_fn( # FLAGS.dev_file, # _decode_record, # name_to_features, params) # [_, eval_loss, # eval_per_example_loss, eval_logits] = model_eval_fn(eval_features, [], tf.estimator.ModeKeys.EVAL) # eval_dict = metric_fn(eval_features, eval_logits, eval_loss) # sess.run(tf.local_variables_initializer()) eval_finial_dict = eval_fn(eval_dict) if hvd.rank() == 0: pkl.dump( eval_finial_dict, open( FLAGS.model_output + "/eval_dict_{}.pkl".format(steps), "wb")) return eval_finial_dict def eval_fn(result): i = 0 total_accuracy = 0 eval_total_dict = {} while True: try: eval_result = sess.run(result) for key in eval_result: if key not in eval_total_dict: if key in ["pred_label", "label_ids"]: eval_total_dict[key] = [] eval_total_dict[key].extend(eval_result[key]) if key in ["accuracy", "loss"]: eval_total_dict[key] = 0.0 eval_total_dict[key] += eval_result[key] else: if key in ["pred_label", "label_ids"]: eval_total_dict[key].extend(eval_result[key]) if key in ["accuracy", "loss"]: eval_total_dict[key] += eval_result[key] i += 1 # if i == 100: # break except tf.errors.OutOfRangeError: print("End of dataset") break label_id = eval_total_dict["label_ids"] pred_label = eval_total_dict["pred_label"] result = classification_report(label_id, pred_label, target_names=list( label_dict["label2id"].keys())) print(result) eval_total_dict["classification_report"] = result return eval_total_dict def train_fn(op_dict): i = 0 cnt = 0 loss_dict = {} monitoring_train = [] monitoring_eval = [] while True: try: train_result = sess.run(op_dict) for key in train_result: if key == "train_op": continue else: if np.isnan(train_result[key]): print(train_loss, "get nan loss") break else: if key in loss_dict: loss_dict[key] += train_result[key] else: loss_dict[key] = train_result[key] i += 1 cnt += 1 if np.mod(i, num_storage_steps) == 0: string = "" for key in loss_dict: tmp = key + " " + str(loss_dict[key] / cnt) + "\t" string += tmp print(string) monitoring_train.append(loss_dict) if hvd.rank() == 0: model_io_fn.save_model( sess, FLAGS.model_output + "/model_{}.ckpt".format( int(i / num_storage_steps))) print("==successful storing model=={}".format( int(i / num_storage_steps))) cnt = 0 # eval_finial_dict = run_eval(int(i/num_storage_steps)) # monitoring_eval.append(eval_finial_dict) for key in loss_dict: loss_dict[key] = 0.0 except tf.errors.OutOfRangeError: if hvd.rank() == 0: import _pickle as pkl pkl.dump( { "train": monitoring_train, "eval": monitoring_eval }, open(FLAGS.model_output + "/monitoring.pkl", "wb")) break print("===========begin to train============") train_fn(train_dict) if hvd.rank() == 0: model_io_fn.save_model(sess, FLAGS.model_output + "/model.ckpt") print("===========begin to eval============") eval_finial_dict = run_eval("final")
def model_fn(features, labels, mode): label_ids = features["label_ids"] model_lst = [] for index, name in enumerate(input_name): if index > 0: reuse = True else: reuse = model_reuse model_lst.append( base_model(model_config, features, labels, mode, name, reuse=reuse)) if mode == tf.estimator.ModeKeys.TRAIN: hidden_dropout_prob = model_config.hidden_dropout_prob attention_probs_dropout_prob = model_config.attention_probs_dropout_prob dropout_prob = model_config.dropout_prob else: hidden_dropout_prob = 0.0 attention_probs_dropout_prob = 0.0 dropout_prob = 0.0 assert len(model_lst) == len(input_name) if model_io_config.fix_lm == True: scope = model_config.scope + "_finetuning" else: scope = model_config.scope with tf.variable_scope(scope, reuse=model_reuse): try: label_ratio_table = tf.get_variable( name="label_ratio", initializer=tf.constant(label_tensor), trainable=False) ratio_weight = tf.nn.embedding_lookup(label_ratio_table, label_ids) print("==applying class weight==") except: ratio_weight = None seq_output_lst = [model.get_pooled_output() for model in model_lst] [loss, per_example_loss, logits] = classifier.order_classifier(model_config, seq_output_lst, num_labels, label_ids, dropout_prob, ratio_weight) # model_io_fn = model_io.ModelIO(model_io_config) pretrained_tvars = model_io_fn.get_params(model_config.scope) if load_pretrained: model_io_fn.load_pretrained(pretrained_tvars, init_checkpoint) tvars = model_io_fn.get_params(scope) if mode == tf.estimator.ModeKeys.TRAIN: model_io_fn.print_params(tvars, string=", trainable params") update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): optimizer_fn = optimizer.Optimizer(opt_config) train_op = optimizer_fn.get_train_op( loss, tvars, opt_config.init_lr, opt_config.num_train_steps) return [train_op, loss, per_example_loss, logits] else: model_io_fn.print_params(tvars, string=", trainable params") return [loss, loss, per_example_loss, logits]
def main(_): graph = tf.Graph() from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score with graph.as_default(): import json hvd.init() sess_config = tf.ConfigProto() sess_config.gpu_options.visible_device_list = str(hvd.local_rank()) # config = json.load(open("/data/xuht/bert/chinese_L-12_H-768_A-12/bert_config.json", "r")) config = json.load(open(FLAGS.config_file, "r")) init_checkpoint = FLAGS.init_checkpoint print("===init checkoutpoint==={}".format(init_checkpoint)) import json label_dict = json.load(open(FLAGS.label_id)) # init_checkpoint = "/data/xuht/bert/chinese_L-12_H-768_A-12/bert_model.ckpt" # init_checkpoint = "/data/xuht/concat/model_1/oqmrc.ckpt" config = Bunch(config) config.use_one_hot_embeddings = True config.scope = "bert" config.dropout_prob = 0.1 config.label_type = "single_label" # config.loss = "focal_loss" # os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu_id sess = tf.Session(config=sess_config) train_size = int(FLAGS.train_size / hvd.size()) num_train_steps = int(train_size / FLAGS.batch_size * FLAGS.epoch) num_warmup_steps = int(num_train_steps * 0.01) num_storage_steps = int(train_size / FLAGS.batch_size) print(num_train_steps, num_warmup_steps, "=============") opt_config = Bunch({ "init_lr": (2e-5 / hvd.size()), "num_train_steps": num_train_steps, "num_warmup_steps": num_warmup_steps }) model_io_config = Bunch({"fix_lm": False}) model_io_fn = model_io.ModelIO(model_io_config) optimizer_fn = optimizer.Optimizer(opt_config) num_choice = FLAGS.num_classes max_seq_length = FLAGS.max_length # model_train_fn = bert_classifier.classifier_model_fn_builder(config, num_choice, init_checkpoint, # reuse=None, # load_pretrained=True, # model_io_fn=model_io_fn, # optimizer_fn=optimizer_fn, # model_io_config=model_io_config, # opt_config=opt_config) model_eval_fn = bert_classifier.classifier_model_fn_builder( config, num_choice, init_checkpoint, reuse=None, load_pretrained=True, model_io_fn=model_io_fn, optimizer_fn=optimizer_fn, model_io_config=model_io_config, opt_config=opt_config) def metric_fn(features, logits, loss): print(logits.get_shape(), "===logits shape===") pred_label = tf.argmax(logits, axis=-1, output_type=tf.int32) prob = tf.nn.softmax(logits) accuracy = correct = tf.equal( tf.cast(pred_label, tf.int32), tf.cast(features["label_ids"], tf.int32)) accuracy = tf.reduce_mean(tf.cast(correct, tf.float32)) return { "accuracy": accuracy, "loss": loss, "pred_label": pred_label, "label_ids": features["label_ids"] } name_to_features = { "input_ids": tf.FixedLenFeature([max_seq_length], tf.int64), "input_mask": tf.FixedLenFeature([max_seq_length], tf.int64), "segment_ids": tf.FixedLenFeature([max_seq_length], tf.int64), "label_ids": tf.FixedLenFeature([], tf.int64), } def _decode_record(record, name_to_features): """Decodes a record to a TensorFlow example. """ example = tf.parse_single_example(record, name_to_features) # tf.Example only supports tf.int64, but the TPU only supports tf.int32. # So cast all int64 to int32. for name in list(example.keys()): t = example[name] if t.dtype == tf.int64: t = tf.to_int32(t) example[name] = t return example params = Bunch({}) params.epoch = FLAGS.epoch params.batch_size = FLAGS.batch_size # train_features = tf_data_utils.train_input_fn("/data/xuht/wsdm19/data/train.tfrecords", # _decode_record, name_to_features, params) # eval_features = tf_data_utils.eval_input_fn("/data/xuht/wsdm19/data/dev.tfrecords", # _decode_record, name_to_features, params) # train_features = tf_data_utils.train_input_fn(FLAGS.train_file, # _decode_record, name_to_features, params) eval_features = tf_data_utils.eval_input_fn(FLAGS.dev_file, _decode_record, name_to_features, params) # [train_op, train_loss, train_per_example_loss, train_logits] = model_train_fn(train_features, [], tf.estimator.ModeKeys.TRAIN) [_, eval_loss, eval_per_example_loss, eval_logits] = model_eval_fn(eval_features, [], tf.estimator.ModeKeys.EVAL) result = metric_fn(eval_features, eval_logits, eval_loss) model_io_fn.set_saver() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess.run(init_op) model_io_fn.load_model(sess, init_checkpoint) sess.run(hvd.broadcast_global_variables(0)) def eval_fn(result): i = 0 total_accuracy = 0 label, label_id = [], [] while True: try: eval_result = sess.run(result) total_accuracy += eval_result["accuracy"] label_id.extend(eval_result["label_ids"]) label.extend(eval_result["pred_label"]) i += 1 except tf.errors.OutOfRangeError: print("End of dataset") break f1 = f1_score(label_id, label, average="macro") accuracy = accuracy_score(label_id, label) print("test accuracy accuracy {} {} f1 {}".format( total_accuracy / i, accuracy, f1)) return total_accuracy / i, f1 if hvd.rank() == 0: print("===========begin to eval============") accuracy, f1 = eval_fn(result) print("==accuracy {} f1 {}==".format(accuracy, f1))
def model_fn(features, labels, mode): label_ids = features["label_ids"] model_lst = [] for index, name in enumerate(input_name): if index > 0: reuse = True else: reuse = model_reuse model_lst.append( base_model(model_config, features, labels, mode, name, reuse=reuse)) if mode == tf.estimator.ModeKeys.TRAIN: hidden_dropout_prob = model_config.hidden_dropout_prob attention_probs_dropout_prob = model_config.attention_probs_dropout_prob dropout_prob = model_config.dropout_prob else: hidden_dropout_prob = 0.0 attention_probs_dropout_prob = 0.0 dropout_prob = 0.0 assert len(model_lst) == len(input_name) if model_io_config.fix_lm == True: scope = model_config.scope + "_finetuning" else: scope = model_config.scope with tf.variable_scope(scope, reuse=model_reuse): try: label_ratio_table = tf.get_variable( name="label_ratio", shape=[ num_labels, ], initializer=tf.constant(label_tensor), trainable=False) ratio_weight = tf.nn.embedding_lookup(label_ratio_table, label_ids) except: ratio_weight = None seq_output_lst = [model.get_pooled_output() for model in model_lst] repres = seq_output_lst[0] + seq_output_lst[1] final_hidden_shape = bert_utils.get_shape_list(repres, expected_rank=2) z_mean = tf.layers.dense(repres, final_hidden_shape[1], name="z_mean") z_log_var = tf.layers.dense(repres, final_hidden_shape[1], name="z_log_var") print("=======applying vib============") if mode == tf.estimator.ModeKeys.TRAIN: print("====applying vib====") vib_connector = vib.VIB(vib_config) [kl_loss, latent_vector ] = vib_connector.build_regularizer([z_mean, z_log_var]) [loss, per_example_loss, logits] = classifier.classifier(model_config, latent_vector, num_labels, label_ids, dropout_prob, ratio_weight) loss += tf.reduce_mean(kl_loss) else: print("====applying z_mean for prediction====") [loss, per_example_loss, logits] = classifier.classifier(model_config, z_mean, num_labels, label_ids, dropout_prob, ratio_weight) # model_io_fn = model_io.ModelIO(model_io_config) pretrained_tvars = model_io_fn.get_params(model_config.scope) if load_pretrained: model_io_fn.load_pretrained(pretrained_tvars, init_checkpoint) tvars = model_io_fn.get_params(scope) if mode == tf.estimator.ModeKeys.TRAIN: model_io_fn.print_params(tvars, string=", trainable params") update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): optimizer_fn = optimizer.Optimizer(opt_config) train_op = optimizer_fn.get_train_op( loss, tvars, opt_config.init_lr, opt_config.num_train_steps) return [train_op, loss, per_example_loss, logits] else: model_io_fn.print_params(tvars, string=", trainable params") return [loss, loss, per_example_loss, logits]
def model_fn(features, labels, mode): label_ids = features["label_ids"] model = base_model(model_config, features, labels, mode, input_name, reuse=model_reuse, perturbation=None) if mode == tf.estimator.ModeKeys.TRAIN: hidden_dropout_prob = model_config.hidden_dropout_prob attention_probs_dropout_prob = model_config.attention_probs_dropout_prob dropout_prob = model_config.dropout_prob else: hidden_dropout_prob = 0.0 attention_probs_dropout_prob = 0.0 dropout_prob = 0.0 # assert len(model_lst) == len(input_name) if model_io_config.fix_lm == True: scope = model_config.scope + "_finetuning" else: scope = model_config.scope def build_discriminator(model, scope, reuse): with tf.variable_scope(scope, reuse=reuse): try: label_ratio_table = tf.get_variable( name="label_ratio", initializer=tf.constant(label_tensor), trainable=False) ratio_weight = tf.nn.embedding_lookup( label_ratio_table, label_ids) print("==applying class weight==") except: ratio_weight = None (loss, per_example_loss, logits) = classifier.classifier(model_config, model.get_pooled_output(), num_labels, label_ids, dropout_prob, ratio_weight) return loss, per_example_loss, logits [loss, per_example_loss, logits] = build_discriminator(model, scope, model_reuse) if mode == tf.estimator.ModeKeys.TRAIN: pretrained_tvars = model_io_fn.get_params( model_config.scope, not_storage_params=not_storage_params) if load_pretrained: tf.logging.info(" load pre-trained base model ") print(" load pre-trained base model ") model_io_fn.load_pretrained(pretrained_tvars, init_checkpoint, exclude_scope=exclude_scope) tvars = pretrained_tvars model_io_fn.set_saver(var_lst=tvars) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): optimizer_fn = optimizer.Optimizer(opt_config) optimizer_fn.get_opt(opt_config.init_lr, opt_config.num_train_steps) perturb = get_perturbation(model_config, optimizer_fn.opt, model.embedding_output_word, loss, tvars) adv_model = base_model(model_config, features, labels, mode, input_name, reuse=True, perturbation=perturb) [adv_loss, adv_per_example_loss, adv_logits] = build_discriminator(adv_model, scope, True) total_loss = adv_loss + loss total_train_op = optimizer_fn.get_train_op_v1( total_loss, tvars) return [total_train_op, total_loss, per_example_loss, logits] else: model_io_fn.set_saver() return [loss, loss, per_example_loss, logits]
def model_fn(features, labels, mode): input_ids = features["input_ids"] input_mask = features["input_mask"] segment_ids = features["segment_ids"] label_ids = features["label_ids"] input_shape = bert_utils.get_shape_list(input_ids, expected_rank=3) batch_size = input_shape[0] choice_num = input_shape[1] seq_length = input_shape[2] input_ids = tf.reshape(input_ids, [batch_size * choice_num, seq_length]) input_mask = tf.reshape(input_mask, [batch_size * choice_num, seq_length]) segment_ids = tf.reshape(segment_ids, [batch_size * choice_num, seq_length]) if mode == tf.estimator.ModeKeys.TRAIN: hidden_dropout_prob = model_config.hidden_dropout_prob attention_probs_dropout_prob = model_config.attention_probs_dropout_prob dropout_prob = model_config.dropout_prob else: hidden_dropout_prob = 0.0 attention_probs_dropout_prob = 0.0 dropout_prob = 0.0 model = bert.Bert(model_config) model.build_embedder(input_ids, segment_ids, hidden_dropout_prob, attention_probs_dropout_prob, reuse=reuse) model.build_encoder(input_ids, input_mask, hidden_dropout_prob, attention_probs_dropout_prob, reuse=reuse) model.build_pooler(reuse=reuse) if model_io_config.fix_lm == True: scope = model_config.scope + "_finetuning" else: scope = model_config.scope with tf.variable_scope(scope, reuse=reuse): (loss, per_example_loss, logits) = classifier.multi_choice_classifier( model_config, model.get_pooled_output(), num_labels, label_ids, dropout_prob) # model_io_fn = model_io.ModelIO(model_io_config) pretrained_tvars = model_io_fn.get_params(model_config.scope) if load_pretrained: model_io_fn.load_pretrained(pretrained_tvars, init_checkpoint, exclude_scope=exclude_scope) tvars = model_io_fn.get_params(scope, not_storage_params=not_storage_params) model_io_fn.set_saver(var_lst=tvars) if mode == tf.estimator.ModeKeys.TRAIN: model_io_fn.print_params(tvars, string=", trainable params") update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): optimizer_fn = optimizer.Optimizer(opt_config) train_op = optimizer_fn.get_train_op( loss, tvars, opt_config.init_lr, opt_config.num_train_steps) return [train_op, loss, per_example_loss, logits] else: model_io_fn.print_params(tvars, string=", trainable params") return [loss, loss, per_example_loss, logits]
def main(_): hvd.init() graph = tf.Graph() sess_config = tf.ConfigProto() sess_config.gpu_options.visible_device_list = str(hvd.local_rank()) from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score with graph.as_default(): import json tokenizer = tokenization.FullTokenizer(vocab_file=FLAGS.vocab_file, do_lower_case=True) classifier_data_api = classifier_processor.PiarChoiceProcessor() eval_examples = classifier_data_api.get_test_examples( FLAGS.eval_data_file, FLAGS.lang) print(eval_examples[0].guid) label_id = json.load(open(FLAGS.label_id, "r")) num_choice = 3 max_seq_length = FLAGS.max_length write_to_tfrecords.convert_classifier_examples_to_features( eval_examples, label_id["label2id"], max_seq_length, tokenizer, FLAGS.output_file) config = json.load(open(FLAGS.config_file, "r")) init_checkpoint = FLAGS.init_checkpoint print("===init checkoutpoint==={}".format(init_checkpoint)) config = Bunch(config) config.use_one_hot_embeddings = True config.scope = "bert" config.dropout_prob = 0.2 config.label_type = "single_label" # os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu_id sess = tf.Session(config=sess_config) train_size = int(FLAGS.train_size / hvd.size()) num_train_steps = int(train_size / FLAGS.batch_size * FLAGS.epoch) num_warmup_steps = int(num_train_steps * 0.01) num_storage_steps = int(train_size / FLAGS.batch_size) print(num_train_steps, num_warmup_steps, "=============") opt_config = Bunch({ "init_lr": (2e-5 / hvd.size()), "num_train_steps": num_train_steps, "num_warmup_steps": num_warmup_steps }) model_io_config = Bunch({"fix_lm": False}) model_io_fn = model_io.ModelIO(model_io_config) optimizer_fn = optimizer.Optimizer(opt_config) num_choice = FLAGS.num_classes max_seq_length = FLAGS.max_length model_eval_fn = bert_classifier.classifier_model_fn_builder( config, num_choice, init_checkpoint, reuse=None, load_pretrained=True, model_io_fn=model_io_fn, optimizer_fn=optimizer_fn, model_io_config=model_io_config, opt_config=opt_config) def metric_fn(features, logits): print(logits.get_shape(), "===logits shape===") pred_label = tf.argmax(logits, axis=-1, output_type=tf.int32) prob = tf.exp(tf.nn.log_softmax(logits)) return { "pred_label": pred_label, "qas_id": features["qas_id"], "prob": prob } name_to_features = { "input_ids": tf.FixedLenFeature([max_seq_length], tf.int64), "input_mask": tf.FixedLenFeature([max_seq_length], tf.int64), "segment_ids": tf.FixedLenFeature([max_seq_length], tf.int64), "label_ids": tf.FixedLenFeature([], tf.int64), "qas_id": tf.FixedLenFeature([], tf.int64), } def _decode_record(record, name_to_features): """Decodes a record to a TensorFlow example. """ example = tf.parse_single_example(record, name_to_features) # tf.Example only supports tf.int64, but the TPU only supports tf.int32. # So cast all int64 to int32. for name in list(example.keys()): t = example[name] if t.dtype == tf.int64: t = tf.to_int32(t) example[name] = t return example params = Bunch({}) params.epoch = 2 params.batch_size = 32 eval_features = tf_data_utils.eval_input_fn(FLAGS.output_file, _decode_record, name_to_features, params) [_, eval_loss, eval_per_example_loss, eval_logits] = model_eval_fn(eval_features, [], tf.estimator.ModeKeys.EVAL) result = metric_fn(eval_features, eval_logits) model_io_fn.set_saver() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess.run(init_op) def eval_fn(result): i = 0 pred_label, qas_id, prob = [], [], [] while True: try: eval_result = sess.run(result) pred_label.extend(eval_result["pred_label"].tolist()) qas_id.extend(eval_result["qas_id"].tolist()) prob.extend(eval_result["prob"].tolist()) i += 1 except tf.errors.OutOfRangeError: print("End of dataset") break return pred_label, qas_id, prob print("===========begin to eval============") [pred_label, qas_id, prob] = eval_fn(result) result = dict(zip(qas_id, pred_label)) print(FLAGS.result_file.split(".")) tmp_output = FLAGS.result_file.split(".")[0] + ".json" print(tmp_output, "===temp output===") json.dump({ "id": qas_id, "label": pred_label, "prob": prob }, open(tmp_output, "w")) print(len(result), "=====valid result======") import pandas as pd df = pd.read_csv(FLAGS.eval_data_file) output = {} for index in range(df.shape[0]): output[df.loc[index]["id"]] = "" final_output = [] cnt = 0 for key in output: if key in result: final_output.append({ "Id": key, "Category": label_id["id2label"][str(result[key])] }) cnt += 1 else: final_output.append({"Id": key, "Category": "unrelated"}) df_out = pd.DataFrame(final_output) df_out.to_csv(FLAGS.result_file) print(len(output), cnt, len(final_output), "======num of results from model==========")
def main(_): hvd.init() sess_config = tf.ConfigProto() sess_config.gpu_options.visible_device_list = str(hvd.local_rank()) graph = tf.Graph() from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score with graph.as_default(): import json # config = json.load(open("/data/xuht/bert/chinese_L-12_H-768_A-12/bert_config.json", "r")) config = json.load(open(FLAGS.config_file, "r")) init_checkpoint = FLAGS.init_checkpoint print("===init checkoutpoint==={}".format(init_checkpoint)) config = Bunch(config) config.use_one_hot_embeddings = True config.scope = "bert" config.dropout_prob = 0.1 config.label_type = "single_label" config.lm_ratio = 0.0 config.task_ratio = 1.0 json.dump(config, open(FLAGS.model_output+"/config.json", "w")) init_lr = 1e-5 if FLAGS.if_shard == "0": train_size = FLAGS.train_size epoch = int(FLAGS.epoch / hvd.size()) elif FLAGS.if_shard == "1": train_size = int(FLAGS.train_size/hvd.size()) epoch = FLAGS.epoch sess = tf.Session(config=sess_config) num_train_steps = int( train_size / FLAGS.batch_size * epoch) num_warmup_steps = int(num_train_steps * 0.1) num_storage_steps = int(train_size / FLAGS.batch_size) print(num_train_steps, num_warmup_steps, "=============") opt_config = Bunch({"init_lr":init_lr/(hvd.size()), "num_train_steps":num_train_steps, "num_warmup_steps":num_warmup_steps}) model_io_config = Bunch({"fix_lm":False}) model_io_fn = model_io.ModelIO(model_io_config) optimizer_fn = optimizer.Optimizer(opt_config) num_choice = FLAGS.num_classes max_seq_length = FLAGS.max_length max_predictions_per_seq = FLAGS.max_predictions_per_seq model_train_fn = classifier_fn.classifier_model_fn_builder(config, num_choice, init_checkpoint, reuse=None, load_pretrained=True, model_io_fn=model_io_fn, optimizer_fn=optimizer_fn, model_io_config=model_io_config, opt_config=opt_config) model_eval_fn = classifier_fn.classifier_model_fn_builder(config, num_choice, init_checkpoint, reuse=True, load_pretrained=True, model_io_fn=model_io_fn, optimizer_fn=optimizer_fn, model_io_config=model_io_config, opt_config=opt_config) name_to_features = { "input_ids": tf.FixedLenFeature([max_seq_length], tf.int64), "input_mask": tf.FixedLenFeature([max_seq_length], tf.int64), "segment_ids": tf.FixedLenFeature([max_seq_length], tf.int64), "masked_lm_positions": tf.FixedLenFeature([max_predictions_per_seq], tf.int64), "masked_lm_ids": tf.FixedLenFeature([max_predictions_per_seq], tf.int64), "masked_lm_weights": tf.FixedLenFeature([max_predictions_per_seq], tf.float32), "label_ids": tf.FixedLenFeature([], tf.int64), } def _decode_record(record, name_to_features): """Decodes a record to a TensorFlow example. """ example = tf.parse_single_example(record, name_to_features) # tf.Example only supports tf.int64, but the TPU only supports tf.int32. # So cast all int64 to int32. for name in list(example.keys()): t = example[name] if t.dtype == tf.int64: t = tf.to_int32(t) example[name] = t return example params = Bunch({}) params.epoch = epoch params.batch_size = FLAGS.batch_size def parse_folder(path): files = os.listdir(path) output = [] for file_name in files: output.append(os.path.join(path, file_name)) random.shuffle(output) return output train_features = tf_data_utils.train_input_fn( parse_folder(FLAGS.train_file), _decode_record, name_to_features, params) train_dict = model_train_fn(train_features, [], tf.estimator.ModeKeys.TRAIN) eval_features = tf_data_utils.eval_input_fn( parse_folder(FLAGS.dev_file), _decode_record, name_to_features, params) eval_dict = model_eval_fn(eval_features, [], tf.estimator.ModeKeys.EVAL) model_io_fn.set_saver() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess.run(init_op) sess.run(hvd.broadcast_global_variables(0)) def eval_fn(op_dict): i = 0 eval_total_dict = {} while True: try: eval_result = sess.run(op_dict) for key in eval_result: if key in ["probabilities", "label_ids"]: if key in eval_total_dict: eval_total_dict[key].extend(eval_result[key]) else: eval_total_dict[key] = [] eval_total_dict[key].extend(eval_result[key]) i += 1 except tf.errors.OutOfRangeError: print("End of dataset") break for key in eval_result: if key not in ["probabilities", "label_ids"]: eval_total_dict[key] = eval_result[key] label_id = eval_total_dict["label_ids"] label = np.argmax(np.array(eval_total_dict["probabilities"]), axis=-1) macro_f1 = f1_score(label_id, label, average="macro") micro_f1 = f1_score(label_id, label, average="micro") accuracy = accuracy_score(label_id, label) print("test accuracy {} macro_f1 score {} micro_f1 {} masked_lm_accuracy {} sentence_f {}".format(accuracy, macro_f1, micro_f1, eval_total_dict["masked_lm_accuracy"], eval_total_dict["sentence_f"])) return eval_total_dict def run_eval(steps): import _pickle as pkl eval_features = tf_data_utils.eval_input_fn( parse_folder(FLAGS.dev_file), _decode_record, name_to_features, params) eval_dict = model_eval_fn(eval_features, [], tf.estimator.ModeKeys.EVAL) sess.run(tf.local_variables_initializer()) eval_finial_dict = eval_fn(eval_dict) if hvd.rank() == 0: pkl.dump(eval_finial_dict, open(FLAGS.model_output+"/eval_dict_{}.pkl".format(steps), "wb")) return eval_finial_dict def train_fn(op_dict): i = 0 cnt = 0 loss_dict = {} monitoring_train = [] monitoring_eval = [] while True: try: train_result = sess.run(op_dict) for key in train_result: if key == "train_op": continue else: if np.isnan(train_result[key]): print(train_loss, "get nan loss") break else: if key in loss_dict: loss_dict[key] += train_result[key] else: loss_dict[key] = train_result[key] i += 1 cnt += 1 if np.mod(i, num_storage_steps) == 0: string = "" for key in loss_dict: tmp = key + " " + str(loss_dict[key]/cnt) + "\t" string += tmp print(string) monitoring_train.append(loss_dict) eval_finial_dict = run_eval(int(i/num_storage_steps)) monitoring_eval.append(eval_finial_dict) for key in loss_dict: loss_dict[key] = 0.0 if hvd.rank() == 0: model_io_fn.save_model(sess, FLAGS.model_output+"/model_{}.ckpt".format(int(i/num_storage_steps))) print("==successful storing model=={}".format(int(i/num_storage_steps))) cnt = 0 except tf.errors.OutOfRangeError: if hvd.rank() == 0: import _pickle as pkl pkl.dump({"train":monitoring_train, "eval":monitoring_eval}, open(FLAGS.model_output+"/monitoring.pkl", "wb")) break print("===========begin to train============") train_fn(train_dict) if hvd.rank() == 0: model_io_fn.save_model(sess, FLAGS.model_output+"/model.ckpt") print("===========begin to eval============") eval_finial_dict = run_eval("final")