def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--data-dir", required=True, help="Location of data files (model weights, etc).") parser.add_argument("--model-name", required=True, help="The name of the model being fine-tuned.") parser.add_argument("--hparams", default="{}", help="JSON dict of model hyperparameters.") # IBO parser.add_argument("--predict", action='store_true', help="prediction mode") args = parser.parse_args() if args.hparams.endswith(".json"): hparams = utils.load_json(args.hparams) else: hparams = json.loads(args.hparams) tf.logging.set_verbosity(tf.logging.ERROR) if args.predict: predict( configure_finetuning.FinetuningConfig(args.model_name, args.data_dir, **hparams)) else: run_finetuning( configure_finetuning.FinetuningConfig(args.model_name, args.data_dir, **hparams))
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--data-dir", required=True, help="Location of data files (model weights, etc).") parser.add_argument("--model-name", required=True, help="The name of the model being fine-tuned.") parser.add_argument("--hparams", default="{}", help="JSON dict of model hyperparameters.") parser.add_argument( "--init-from-checkpoint", required=True, help="Directory of trained finetune model to run error analysis") args = parser.parse_args() if args.hparams.endswith(".json"): hparams = utils.load_json(args.hparams) else: hparams = json.loads(args.hparams) tf.logging.set_verbosity(tf.logging.ERROR) pretraining_config = {'model_dir': args.init_from_checkpoint} run_analyse( configure_finetuning.FinetuningConfig(args.model_name, args.data_dir, **hparams), pretraining_config)
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--data-dir", required=True, help="Location of data files (model weights, etc).") parser.add_argument("--model-name", required=True, help="The name of the model being fine-tuned.") parser.add_argument("--hparams", default="{}", help="JSON dict of model hyperparameters.") args = parser.parse_args() if args.hparams.endswith(".json"): hparams = utils.load_json(args.hparams) else: hparams = json.loads(args.hparams) tf.logging.set_verbosity(tf.logging.ERROR) config = configure_finetuning.FinetuningConfig(args.model_name, args.data_dir, **hparams) objective = create_obj_fun(config) trials = Trials() best = fmin(objective, space={ 'num_epochs': hp.choice('num_epochs', [3, 5, 10, 15, 20]), 'lr': hp.choice('lr', [1e-5, 5e-5, 1e-4, 5e-4]), 'batch_size': hp.choice('batch_size', [12, 24, 32, 64]) }, algo=tpe.suggest, max_evals=25, trials=trials) print('Best:', best) print("Trials") for trial in trials.trials: print(trial)
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--data_dir", required=True,help="Location of data files (model weights, etc).") parser.add_argument("--model_name", required=True,help="The name of the model being fine-tuned.") parser.add_argument("--corpus", required=True,help="The corpus of model") parser.add_argument("--model_path", required=True,help="the pretrained model path") parser.add_argument("--output_path", required=True,help="the fine-tune model") parser.add_argument("--tfrecords_dir",required=True,help="tfrecords file path") parser.add_argument("--gpu",required=True,help="gpu devices") parser.add_argument("--task_names",required=True,help="task name to train") parser.add_argument("--learning_rate",type=float,default=0.0001,help="the learning_rate") parser.add_argument("--num_train_epochs",type=int,default=3,help="the epochs to train") parser.add_argument("--save_checkpoints_steps",type=int,default=1000,help="save the model") parser.add_argument("--iterations_per_loop",type=int,default=1000,help="iterations_per_loop") parser.add_argument("--train_batch_size",type=int,default=2,help="train_batch_size") parser.add_argument("--eval_batch_size",type=int,default=2,help="eval_batch_size") parser.add_argument("--predict_batch_size",type=int,default=2,help="predict_batch_size") parser.add_argument("--max_seq_length",type=int,default=512,help="the max length for sequence") parser.add_argument('--do_train', action='store_true',help='train the model') parser.add_argument('--do_eval', action='store_true',help='eval the model') parser.add_argument('--do_predict', action='store_true',help='eval the model') parser.add_argument("--hparams", default="{}",help="JSON dict of model hyperparameters.") args = parser.parse_args() if args.hparams.endswith(".json"): hparams = utils.load_json(args.hparams) else: hparams = json.loads(args.hparams) tf.logging.set_verbosity(tf.logging.ERROR) #model_name,model_path,corpus,data_dir,tfrecords_dir,output_dir run_finetuning(configure_finetuning.FinetuningConfig(args=args))
def construct_model(): tf.compat.v1.disable_eager_execution() batch_size = 1 iterations = 10 tf_datatype = tf.int32 np_datatype = np.int32 graph_features = {} graph_features["input_ids"] = tf.placeholder(dtype=tf_datatype, shape=[batch_size, 128], name="input_ids") graph_features["input_mask"] = tf.placeholder(dtype=tf_datatype, shape=[batch_size, 128], name="input_mask") graph_features["segment_ids"] = tf.placeholder(dtype=tf_datatype, shape=[batch_size, 128], name="token_type_ids") #graph_features["task_id"] = tf.placeholder(dtype=tf_datatype, shape=(batch_size,), name="task_id") #graph_features["cola_label_ids"] = tf.placeholder(dtype=tf_datatype, shape=(batch_size,), name="cola_label_ids") #graph_features["cola_eid"] = tf.placeholder(dtype=tf_datatype, shape=(batch_size,), name="cola_eid") features = {} features["input_ids"] = np.random.rand(batch_size, 128).astype(np_datatype) features["input_mask"] = np.random.rand(batch_size, 128).astype(np_datatype) features["segment_ids"] = np.random.rand(batch_size, 128).astype(np_datatype) #features["task_id"] = np.random.rand(batch_size).astype(np_datatype) #features["cola_label_ids"] = np.random.rand(batch_size).astype(np_datatype) #features["cola_eid"] = np.random.rand(batch_size).astype(np_datatype) features_feed_dict = { graph_features[key]: features[key] for key in graph_features } print(features_feed_dict) features_use = graph_features features_use["task_id"] = tf.constant(0, dtype=tf_datatype, shape=(batch_size, ), name="task_id") features_use["cola_label_ids"] = tf.constant(0, dtype=tf_datatype, shape=(batch_size, ), name="cola_label_ids") features_use["cola_eid"] = tf.constant(0, dtype=tf_datatype, shape=(batch_size, ), name="cola_eid") param = {"model_size": "medium-small", "task_names": ["cola"]} config = configure_finetuning.FinetuningConfig( model_name="convbert_medium-small", data_dir="./", **param) task = task_builder.get_tasks(config) print("Getting tasks:".format(task)) is_training = False nums_steps = 0 model = FinetuningModel(config, task, is_training, features_use, nums_steps) outputs = model.outputs out_dict = {} for tks in task: out_dict = outputs[tks.name] output_names = [] for key in out_dict: output_names.append(out_dict[key].name) print(out_dict[key].name) run_op_list = [] outputs_names_with_port = output_names outputs_names_without_port = [ name.split(":")[0] for name in outputs_names_with_port ] for index in range(len(outputs_names_without_port)): run_op_list.append(outputs_names_without_port[index]) print(run_op_list) inputs_names_with_port = [ graph_features[key].name for key in graph_features ] cfg = tf.ConfigProto() cfg.gpu_options.allow_growth = True with tf.Session(config=cfg) as sess: sess.run(tf.global_variables_initializer()) for i in range(iterations): sess.run(run_op_list, feed_dict=features_feed_dict) tf_time_sum = 0 a = datetime.now() for i in range(iterations): tf_result = sess.run(run_op_list, feed_dict=features_feed_dict) b = datetime.now() tf_time_sum = (b - a).total_seconds() tf_time = "[INFO] TF execution time: " + str( tf_time_sum * 1000 / iterations) + " ms" print(tf_time) frozen_graph = tf.graph_util.convert_variables_to_constants( sess, sess.graph_def, outputs_names_without_port) # frozen_graph = tf.graph_util.remove_training_nodes(frozen_graph) # save frozen model with open("ConvBert.pb", "wb") as ofile: ofile.write(frozen_graph.SerializeToString()) exit(0) inputs_names_with_port.remove("task_id:0") onnx_model_file = "ConvBert.onnx" command = "python3 -m tf2onnx.convert --input ConvBert.pb --output %s --fold_const --opset 12 --verbose" % onnx_model_file command += " --inputs " for name in inputs_names_with_port: command += "%s," % name command = command[:-1] + " --outputs " for name in outputs_names_with_port: command += "%s," % name command = command[:-1] os.system(command) print(command) #exit(0) #do not convert now, it needs to modify onehot layer #command = "trtexec --onnx=ConvBert.onnx --verbose" #os.system(command) print(command)
def main(): tf.set_random_seed(1234) np.random.seed(0) batch_size = 1 tf_datatype = tf.int32 np_datatype = np.int32 iterations = 10 features_ph = {} features_ph["input_ids"] = tf.placeholder(dtype=tf_datatype, shape=[batch_size, 128], name="input_ids") features_ph["input_mask"] = tf.placeholder(dtype=tf_datatype, shape=[batch_size, 128], name="input_mask") features_ph["token_type_ids"] = tf.placeholder(dtype=tf_datatype, shape=[batch_size, 128], name="token_type_ids") features_data = {} features_data["input_ids"] = np.random.rand(batch_size, 128).astype(np_datatype) features_data["input_mask"] = np.random.rand(batch_size, 128).astype(np_datatype) features_data["token_type_ids"] = np.random.rand( batch_size, 128).astype(np_datatype) features_feed_dict = { features_ph[key]: features_data[key] for key in features_ph } finetuning_config = configure_finetuning.FinetuningConfig("ConvBert", "./") bert_config = training_utils.get_bert_config(finetuning_config) bert_model = modeling.BertModel( bert_config=bert_config, is_training=False, input_ids=features_ph["input_ids"], input_mask=features_ph["input_mask"], token_type_ids=features_ph["token_type_ids"]) #outputs_names = "discriminator_predictions/Sigmoid:0,discriminator_predictions/truediv:0,discriminator_predictions/Cast_2:0,discriminator_predictions/truediv_1:0" graph_outputs = bert_model.get_sequence_output() outputs_names = graph_outputs.name print("graph output: ", graph_outputs) run_op_list = [] outputs_names_with_port = outputs_names.split(",") outputs_names_without_port = [ name.split(":")[0] for name in outputs_names_with_port ] for index in range(len(outputs_names_without_port)): run_op_list.append(outputs_names_without_port[index]) inputs_names_with_port = [features_ph[key].name for key in features_ph] # define saver #saver = tf.train.Saver(var_list=tf.trainable_variables()) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()) for i in range(iterations): sess.run(run_op_list, feed_dict=features_feed_dict) tf_time_sum = 0 a = datetime.now() for i in range(iterations): tf_result = sess.run(run_op_list, feed_dict=features_feed_dict) b = datetime.now() tf_time_sum = (b - a).total_seconds() tf_time = "[INFO] TF execution time: " + str( tf_time_sum * 1000 / iterations) + " ms" # tf_result = tf_result.flatten() frozen_graph = tf.graph_util.convert_variables_to_constants( sess, sess.graph_def, outputs_names_without_port) # frozen_graph = tf.graph_util.remove_training_nodes(frozen_graph) # save frozen model with open("ConvBert.pb", "wb") as ofile: ofile.write(frozen_graph.SerializeToString()) # tf.reset_default_graph() # tf.import_graph_def(frozen_graph, name='') # #with tf.Session(config=config) as sess: # sess = tf.Session(config=config) # graph_def = tf_optimize(inputs_names_with_port, outputs_names_without_port, # sess.graph_def, True) # with open("ConvBert_optimized_model.pb", "wb") as ofile: # ofile.write(graph_def.SerializeToString()) onnx_model_file = "ConvBert.onnx" command = "python3 -m tf2onnx.convert --input ConvBert.pb --output %s --fold_const --opset 12 --verbose" % onnx_model_file command += " --inputs " for name in inputs_names_with_port: command += "%s," % name command = command[:-1] + " --outputs " for name in outputs_names_with_port: command += "%s," % name command = command[:-1] os.system(command) print(command) #exit(0) command = "trtexec - -onnx = ConvBert.onnx - -verbose" os.system(command) print(command)