def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--data-dir",
required=True,
help="Location of data files (model weights, etc).")
parser.add_argument("--model-name",
required=True,
help="The name of the model being fine-tuned.")
parser.add_argument("--hparams",
default="{}",
help="JSON dict of model hyperparameters.")
# IBO
parser.add_argument("--predict",
action='store_true',
help="prediction mode")
args = parser.parse_args()
if args.hparams.endswith(".json"):
hparams = utils.load_json(args.hparams)
else:
hparams = json.loads(args.hparams)
tf.logging.set_verbosity(tf.logging.ERROR)
if args.predict:
predict(
configure_finetuning.FinetuningConfig(args.model_name,
args.data_dir, **hparams))
else:
run_finetuning(
configure_finetuning.FinetuningConfig(args.model_name,
args.data_dir, **hparams))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--data-dir",
required=True,
help="Location of data files (model weights, etc).")
parser.add_argument("--model-name",
required=True,
help="The name of the model being fine-tuned.")
parser.add_argument("--hparams",
default="{}",
help="JSON dict of model hyperparameters.")
parser.add_argument(
"--init-from-checkpoint",
required=True,
help="Directory of trained finetune model to run error analysis")
args = parser.parse_args()
if args.hparams.endswith(".json"):
hparams = utils.load_json(args.hparams)
else:
hparams = json.loads(args.hparams)
tf.logging.set_verbosity(tf.logging.ERROR)
pretraining_config = {'model_dir': args.init_from_checkpoint}
run_analyse(
configure_finetuning.FinetuningConfig(args.model_name, args.data_dir,
**hparams), pretraining_config)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--data-dir",
required=True,
help="Location of data files (model weights, etc).")
parser.add_argument("--model-name",
required=True,
help="The name of the model being fine-tuned.")
parser.add_argument("--hparams",
default="{}",
help="JSON dict of model hyperparameters.")
args = parser.parse_args()
if args.hparams.endswith(".json"):
hparams = utils.load_json(args.hparams)
else:
hparams = json.loads(args.hparams)
tf.logging.set_verbosity(tf.logging.ERROR)
config = configure_finetuning.FinetuningConfig(args.model_name,
args.data_dir, **hparams)
objective = create_obj_fun(config)
trials = Trials()
best = fmin(objective,
space={
'num_epochs': hp.choice('num_epochs', [3, 5, 10, 15, 20]),
'lr': hp.choice('lr', [1e-5, 5e-5, 1e-4, 5e-4]),
'batch_size': hp.choice('batch_size', [12, 24, 32, 64])
},
algo=tpe.suggest,
max_evals=25,
trials=trials)
print('Best:', best)
print("Trials")
for trial in trials.trials:
print(trial)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--data_dir", required=True,help="Location of data files (model weights, etc).")
parser.add_argument("--model_name", required=True,help="The name of the model being fine-tuned.")
parser.add_argument("--corpus", required=True,help="The corpus of model")
parser.add_argument("--model_path", required=True,help="the pretrained model path")
parser.add_argument("--output_path", required=True,help="the fine-tune model")
parser.add_argument("--tfrecords_dir",required=True,help="tfrecords file path")
parser.add_argument("--gpu",required=True,help="gpu devices")
parser.add_argument("--task_names",required=True,help="task name to train")
parser.add_argument("--learning_rate",type=float,default=0.0001,help="the learning_rate")
parser.add_argument("--num_train_epochs",type=int,default=3,help="the epochs to train")
parser.add_argument("--save_checkpoints_steps",type=int,default=1000,help="save the model")
parser.add_argument("--iterations_per_loop",type=int,default=1000,help="iterations_per_loop")
parser.add_argument("--train_batch_size",type=int,default=2,help="train_batch_size")
parser.add_argument("--eval_batch_size",type=int,default=2,help="eval_batch_size")
parser.add_argument("--predict_batch_size",type=int,default=2,help="predict_batch_size")
parser.add_argument("--max_seq_length",type=int,default=512,help="the max length for sequence")
parser.add_argument('--do_train', action='store_true',help='train the model')
parser.add_argument('--do_eval', action='store_true',help='eval the model')
parser.add_argument('--do_predict', action='store_true',help='eval the model')
parser.add_argument("--hparams", default="{}",help="JSON dict of model hyperparameters.")
args = parser.parse_args()
if args.hparams.endswith(".json"):
hparams = utils.load_json(args.hparams)
else:
hparams = json.loads(args.hparams)
tf.logging.set_verbosity(tf.logging.ERROR)
#model_name,model_path,corpus,data_dir,tfrecords_dir,output_dir
run_finetuning(configure_finetuning.FinetuningConfig(args=args))
def construct_model():
tf.compat.v1.disable_eager_execution()
batch_size = 1
iterations = 10
tf_datatype = tf.int32
np_datatype = np.int32
graph_features = {}
graph_features["input_ids"] = tf.placeholder(dtype=tf_datatype,
shape=[batch_size, 128],
name="input_ids")
graph_features["input_mask"] = tf.placeholder(dtype=tf_datatype,
shape=[batch_size, 128],
name="input_mask")
graph_features["segment_ids"] = tf.placeholder(dtype=tf_datatype,
shape=[batch_size, 128],
name="token_type_ids")
#graph_features["task_id"] = tf.placeholder(dtype=tf_datatype, shape=(batch_size,), name="task_id")
#graph_features["cola_label_ids"] = tf.placeholder(dtype=tf_datatype, shape=(batch_size,), name="cola_label_ids")
#graph_features["cola_eid"] = tf.placeholder(dtype=tf_datatype, shape=(batch_size,), name="cola_eid")
features = {}
features["input_ids"] = np.random.rand(batch_size, 128).astype(np_datatype)
features["input_mask"] = np.random.rand(batch_size,
128).astype(np_datatype)
features["segment_ids"] = np.random.rand(batch_size,
128).astype(np_datatype)
#features["task_id"] = np.random.rand(batch_size).astype(np_datatype)
#features["cola_label_ids"] = np.random.rand(batch_size).astype(np_datatype)
#features["cola_eid"] = np.random.rand(batch_size).astype(np_datatype)
features_feed_dict = {
graph_features[key]: features[key]
for key in graph_features
}
print(features_feed_dict)
features_use = graph_features
features_use["task_id"] = tf.constant(0,
dtype=tf_datatype,
shape=(batch_size, ),
name="task_id")
features_use["cola_label_ids"] = tf.constant(0,
dtype=tf_datatype,
shape=(batch_size, ),
name="cola_label_ids")
features_use["cola_eid"] = tf.constant(0,
dtype=tf_datatype,
shape=(batch_size, ),
name="cola_eid")
param = {"model_size": "medium-small", "task_names": ["cola"]}
config = configure_finetuning.FinetuningConfig(
model_name="convbert_medium-small", data_dir="./", **param)
task = task_builder.get_tasks(config)
print("Getting tasks:".format(task))
is_training = False
nums_steps = 0
model = FinetuningModel(config, task, is_training, features_use,
nums_steps)
outputs = model.outputs
out_dict = {}
for tks in task:
out_dict = outputs[tks.name]
output_names = []
for key in out_dict:
output_names.append(out_dict[key].name)
print(out_dict[key].name)
run_op_list = []
outputs_names_with_port = output_names
outputs_names_without_port = [
name.split(":")[0] for name in outputs_names_with_port
]
for index in range(len(outputs_names_without_port)):
run_op_list.append(outputs_names_without_port[index])
print(run_op_list)
inputs_names_with_port = [
graph_features[key].name for key in graph_features
]
cfg = tf.ConfigProto()
cfg.gpu_options.allow_growth = True
with tf.Session(config=cfg) as sess:
sess.run(tf.global_variables_initializer())
for i in range(iterations):
sess.run(run_op_list, feed_dict=features_feed_dict)
tf_time_sum = 0
a = datetime.now()
for i in range(iterations):
tf_result = sess.run(run_op_list, feed_dict=features_feed_dict)
b = datetime.now()
tf_time_sum = (b - a).total_seconds()
tf_time = "[INFO] TF execution time: " + str(
tf_time_sum * 1000 / iterations) + " ms"
print(tf_time)
frozen_graph = tf.graph_util.convert_variables_to_constants(
sess, sess.graph_def, outputs_names_without_port)
# frozen_graph = tf.graph_util.remove_training_nodes(frozen_graph)
# save frozen model
with open("ConvBert.pb", "wb") as ofile:
ofile.write(frozen_graph.SerializeToString())
exit(0)
inputs_names_with_port.remove("task_id:0")
onnx_model_file = "ConvBert.onnx"
command = "python3 -m tf2onnx.convert --input ConvBert.pb --output %s --fold_const --opset 12 --verbose" % onnx_model_file
command += " --inputs "
for name in inputs_names_with_port:
command += "%s," % name
command = command[:-1] + " --outputs "
for name in outputs_names_with_port:
command += "%s," % name
command = command[:-1]
os.system(command)
print(command)
#exit(0)
#do not convert now, it needs to modify onehot layer
#command = "trtexec --onnx=ConvBert.onnx --verbose"
#os.system(command)
print(command)
def main():
tf.set_random_seed(1234)
np.random.seed(0)
batch_size = 1
tf_datatype = tf.int32
np_datatype = np.int32
iterations = 10
features_ph = {}
features_ph["input_ids"] = tf.placeholder(dtype=tf_datatype,
shape=[batch_size, 128],
name="input_ids")
features_ph["input_mask"] = tf.placeholder(dtype=tf_datatype,
shape=[batch_size, 128],
name="input_mask")
features_ph["token_type_ids"] = tf.placeholder(dtype=tf_datatype,
shape=[batch_size, 128],
name="token_type_ids")
features_data = {}
features_data["input_ids"] = np.random.rand(batch_size,
128).astype(np_datatype)
features_data["input_mask"] = np.random.rand(batch_size,
128).astype(np_datatype)
features_data["token_type_ids"] = np.random.rand(
batch_size, 128).astype(np_datatype)
features_feed_dict = {
features_ph[key]: features_data[key]
for key in features_ph
}
finetuning_config = configure_finetuning.FinetuningConfig("ConvBert", "./")
bert_config = training_utils.get_bert_config(finetuning_config)
bert_model = modeling.BertModel(
bert_config=bert_config,
is_training=False,
input_ids=features_ph["input_ids"],
input_mask=features_ph["input_mask"],
token_type_ids=features_ph["token_type_ids"])
#outputs_names = "discriminator_predictions/Sigmoid:0,discriminator_predictions/truediv:0,discriminator_predictions/Cast_2:0,discriminator_predictions/truediv_1:0"
graph_outputs = bert_model.get_sequence_output()
outputs_names = graph_outputs.name
print("graph output: ", graph_outputs)
run_op_list = []
outputs_names_with_port = outputs_names.split(",")
outputs_names_without_port = [ name.split(":")[0] for name in outputs_names_with_port ]
for index in range(len(outputs_names_without_port)):
run_op_list.append(outputs_names_without_port[index])
inputs_names_with_port = [features_ph[key].name for key in features_ph]
# define saver
#saver = tf.train.Saver(var_list=tf.trainable_variables())
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
for i in range(iterations):
sess.run(run_op_list, feed_dict=features_feed_dict)
tf_time_sum = 0
a = datetime.now()
for i in range(iterations):
tf_result = sess.run(run_op_list, feed_dict=features_feed_dict)
b = datetime.now()
tf_time_sum = (b - a).total_seconds()
tf_time = "[INFO] TF execution time: " + str(
tf_time_sum * 1000 / iterations) + " ms"
# tf_result = tf_result.flatten()
frozen_graph = tf.graph_util.convert_variables_to_constants(
sess, sess.graph_def, outputs_names_without_port)
# frozen_graph = tf.graph_util.remove_training_nodes(frozen_graph)
# save frozen model
with open("ConvBert.pb", "wb") as ofile:
ofile.write(frozen_graph.SerializeToString())
# tf.reset_default_graph()
# tf.import_graph_def(frozen_graph, name='')
# #with tf.Session(config=config) as sess:
# sess = tf.Session(config=config)
# graph_def = tf_optimize(inputs_names_with_port, outputs_names_without_port,
# sess.graph_def, True)
# with open("ConvBert_optimized_model.pb", "wb") as ofile:
# ofile.write(graph_def.SerializeToString())
onnx_model_file = "ConvBert.onnx"
command = "python3 -m tf2onnx.convert --input ConvBert.pb --output %s --fold_const --opset 12 --verbose" % onnx_model_file
command += " --inputs "
for name in inputs_names_with_port:
command += "%s," % name
command = command[:-1] + " --outputs "
for name in outputs_names_with_port:
command += "%s," % name
command = command[:-1]
os.system(command)
print(command)
#exit(0)
command = "trtexec - -onnx = ConvBert.onnx - -verbose"
os.system(command)
print(command)
