def predict(self, inputs, is_training=False): loss, predict = self(inputs, is_training) return predict model = BERT_NER(param) model.build(input_shape=(4, param.batch_size, param.maxlen)) model.summary() # 构建优化器 optimizer_bert = optim.AdamWarmup( learning_rate=2e-5, # 重要参数 decay_steps=10000, # 重要参数 warmup_steps=1000, ) optimizer_crf = optim.AdamWarmup( learning_rate=1e-3, decay_steps=10000, # 重要参数 warmup_steps=1000, ) # # 初始化参数 bert_init_weights_from_checkpoint(model, model_path, param.num_hidden_layers, pooler=False) # 写入数据 通过check_exist=True参数控制仅在第一次调用时写入
model = TextCNN.TextCNN( maxlen, vocab_size, embedding_dims, class_num, # init_weights, weights_trainable=True) # model = TextCNN.TextCNN(maxlen, vocab_size, embedding_dims, class_num) # 构建优化器 # lr = tf.keras.optimizers.schedules.PolynomialDecay(0.01, decay_steps=18000, # end_learning_rate=0.0001, # cycle=False) optimizer = optim.AdamWarmup(learning_rate=0.01, decay_steps=15000) # 构建损失函数 mask_sparse_categotical_loss = Losess.MaskSparseCategoricalCrossentropy( from_logits=False) f1score = Metric.SparseF1Score(average="macro") precsionscore = Metric.SparsePrecisionScore(average="macro") recallscore = Metric.SparseRecallScore(average="macro") accuarcyscore = Metric.SparseAccuracy() # 保存模型 checkpoint = tf.train.Checkpoint(model=model) manager = tf.train.CheckpointManager(checkpoint, directory="./save", checkpoint_name="model.ckpt",
def call(self, inputs, training=True): logits = self.bilstm(inputs, training) # [B,T,2H] logits, _ = self.att(logits) logits = self.dense(logits) return logits def predict(self,inputs,training=False): out = self(inputs,training) return out model = BilstmAttention(maxlen, vocab_size, embedding_dims, hidden_dim) # 构建优化器 optimizer = optim.AdamWarmup(0.01, decay_steps=15000) # 构建损失函数 mask_sparse_categotical_loss = Losess.MaskSparseCategoricalCrossentropy(from_logits=False) f1score = Metric.SparseF1Score(average="macro") precsionscore = Metric.SparsePrecisionScore(average="macro") recallscore = Metric.SparseRecallScore(average="macro") accuarcyscore = Metric.SparseAccuracy() # 保存模型 checkpoint = tf.train.Checkpoint(model=model) manager = tf.train.CheckpointManager(checkpoint, directory="./save", checkpoint_name="model.ckpt", max_to_keep=3) Batch = 0 for X, token_type_id, input_mask, Y in load.load_train():
def predict(self, inputs, is_training=False): output = self(inputs, is_training=is_training) return output model = ALBERT_NER(param) model.build(input_shape=(3, param.batch_size, param.maxlen)) model.summary() # batch_size增加k学习率也要增加k倍 # optimizer_bert = optim.AdamWarmup(learning_rate=3e-6, # 重要参数 # decay_steps= 0, # 重要参数 # warmup_steps= 0) # 0.1 optimizer_bert = optim.AdamWarmup(learning_rate=1e-6, decay_steps=40000) # 0.1 sparse_categotical_loss = tf.keras.losses.SparseCategoricalCrossentropy( from_logits=False) # 初始化参数 albert_init_weights_from_checkpoint(model, model_path, param.num_hidden_layers, pooler=True) # 写入数据 通过check_exist=True参数控制仅在第一次调用时写入 writer = TFWriter(param.maxlen, vocab_file, modes=["train"], check_exist=False,
def main(_): model = BERT_NER(param) model.build(input_shape=(4, param.batch_size, param.maxlen)) model.summary() # 构建优化器 optimizer_bert = optim.AdamWarmup(learning_rate=2e-5, # 重要参数 decay_steps=10000, # 重要参数 warmup_steps=1000, ) optimizer_crf = optim.AdamWarmup(learning_rate=1e-3, decay_steps=10000, # 重要参数 warmup_steps=1000, ) # # 初始化参数 bert_init_weights_from_checkpoint(model, model_path, param.num_hidden_layers, pooler=False) # 写入数据 通过check_exist=True参数控制仅在第一次调用时写入 writer = TFWriter(param.maxlen, vocab_file, data_dir=FLAGS.data_dir, modes=["train"], check_exist=False) ner_load = TFLoader(param.maxlen, param.batch_size, data_dir=FLAGS.data_dir, epoch=5) # 训练模型 # 使用tensorboard summary_writer = tf.summary.create_file_writer("./tensorboard") # Metrics f1score = Metric.SparseF1Score(average="macro", predict_sparse=True) precsionscore = Metric.SparsePrecisionScore(average="macro", predict_sparse=True) recallscore = Metric.SparseRecallScore(average="macro", predict_sparse=True) accuarcyscore = Metric.SparseAccuracy(predict_sparse=True) # 保存模型 checkpoint = tf.train.Checkpoint(model=model) manager = tf.train.CheckpointManager(checkpoint, directory="./save", checkpoint_name="model.ckpt", max_to_keep=3) # For train model Batch = 0 for X, token_type_id, input_mask, Y in ner_load.load_train(): with tf.GradientTape(persistent=True) as tape: loss, predict = model([X, token_type_id, input_mask, Y]) f1 = f1score(Y, predict) precision = precsionscore(Y, predict) recall = recallscore(Y, predict) accuracy = accuarcyscore(Y, predict) if Batch % 101 == 0: print("Batch:{}\tloss:{:.4f}".format(Batch, loss.numpy())) print("Batch:{}\tacc:{:.4f}".format(Batch, accuracy)) print("Batch:{}\tprecision{:.4f}".format(Batch, precision)) print("Batch:{}\trecall:{:.4f}".format(Batch, recall)) print("Batch:{}\tf1score:{:.4f}".format(Batch, f1)) print("Sentence", writer.convert_id_to_vocab(tf.reshape(X, [-1]).numpy())) print("predict", writer.convert_id_to_label(tf.reshape(predict, [-1]).numpy())) print("label", writer.convert_id_to_label(tf.reshape(Y, [-1]).numpy())) manager.save(checkpoint_number=Batch) with summary_writer.as_default(): tf.summary.scalar("loss", loss, step=Batch) tf.summary.scalar("acc", accuracy, step=Batch) tf.summary.scalar("f1", f1, step=Batch) tf.summary.scalar("precision", precision, step=Batch) tf.summary.scalar("recall", recall, step=Batch) grads_bert = tape.gradient(loss, model.bert.variables + model.dense.variables) grads_crf = tape.gradient(loss, model.crf.variables) optimizer_bert.apply_gradients(grads_and_vars=zip(grads_bert, model.bert.variables + model.dense.variables)) optimizer_crf.apply_gradients(grads_and_vars=zip(grads_crf, model.crf.variables)) Batch += 1 model.save("model_save")
drop_rate = 0.5 epoch = 200 early_stopping = 100 penalty = 0.01 loader = graphloader.GCNLoader(dataset="citeseer", loop=True, features_norm=True) features, adj, y_train, y_val, y_test, train_mask, val_mask, test_mask = loader.load( ) model = GIN.GINLayer(hidden_dim, num_class, drop_rate) # optimizer = tf.keras.optimizers.Adam(0.01) optimizer = optim.AdamWarmup(0.01, 500, weight_decay_rate=0) crossentropy = Losess.MaskCategoricalCrossentropy() accscore = Metric.MaskAccuracy() stop_monitor = EarlyStopping(monitor="loss", patience=early_stopping) # --------------------------------------------------------- # For train for p in range(epoch): t = time.time() with tf.GradientTape() as tape: predict = model(features, adj, training=True) loss = crossentropy(y_train, predict, train_mask) loss += penalty * tf.nn.l2_loss(model.variables[0]) grads = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(
def predict(self, inputs, is_training=False): output = self(inputs, is_training=is_training) return output model = ALBERT_NER(param) model.build(input_shape=(3, param.batch_size, param.maxlen)) model.summary() # batch_size增加k学习率也要增加k倍 # optimizer_bert = optim.AdamWarmup(learning_rate=2e-5, # 重要参数 # decay_steps=10000, # 重要参数 # warmup_steps=1000) # 0.1 optimizer_bert = optim.AdamWarmup(learning_rate=1e-5) # 0.1 sparse_categotical_loss = tf.keras.losses.SparseCategoricalCrossentropy( from_logits=False) # 初始化参数 albert_init_weights_from_checkpoint(model, model_path, param.num_hidden_layers, pooler=False) # 写入数据 通过check_exist=True参数控制仅在第一次调用时写入 writer = TFWriter(param.maxlen, vocab_file, modes=["train"], check_exist=False,