output = tf.math.softmax(pre, axis=-1)
return output
def predict(self, inputs, is_training=False):
output = self(inputs, is_training=is_training)
return output
model = BERT_NER(param)
model.build(input_shape=(3, param.batch_size, param.maxlen))
model.summary()
# 写入数据 通过check_exist=True参数控制仅在第一次调用时写入
writer = TFWriter(param.maxlen, vocab_file,
modes=["valid"], check_exist=False)
ner_load = TFLoader(param.maxlen, param.batch_size, epoch=3)
# Metrics
f1score = Metric.SparseF1Score(average="macro")
precsionscore = Metric.SparsePrecisionScore(average="macro")
recallscore = Metric.SparseRecallScore(average="macro")
accuarcyscore = Metric.SparseAccuracy()
# 保存模型
checkpoint = tf.train.Checkpoint(model=model)
checkpoint.restore(tf.train.latest_checkpoint('./save'))
# For test model
Batch = 0
f1s = []
output = self(inputs, is_training=is_training)
return output
model = ALBERT_NER(param)
model.build(input_shape=(3, param.batch_size, param.maxlen))
model.summary()
# 构建优化器
# 写入数据 通过check_exist=True参数控制仅在第一次调用时写入
writer = TFWriter(param.maxlen,
vocab_file,
modes=["valid"],
check_exist=False,
tokenizer="sentencepiece",
spm_model_file=spm_model_file)
ner_load = TFLoader(param.maxlen, param.batch_size, epoch=8)
# Metrics
f1score = Metric.SparseF1Score(average="macro")
precsionscore = Metric.SparsePrecisionScore(average="macro")
recallscore = Metric.SparseRecallScore(average="macro")
accuarcyscore = Metric.SparseAccuracy()
# 保存模型
checkpoint = tf.train.Checkpoint(model=model)
checkpoint.restore(tf.train.latest_checkpoint('./save'))
# For test model
from nlpgnn.optimizers import optim
from nlpgnn.metrics import Metric, Losess
from nlpgnn.datas.dataloader import TFWriter, TFLoader
maxlen = 128
batch_size = 64
embedding_dims = 100
vocab_file = "Input/vocab.txt"
word2vec = "./corpus/word2vec.vector"
class_num = 2
vocab_size = 30522 # line in vocab.txt
# 写入数据 通过check_exist=True参数控制仅在第一次调用时写入
writer = TFWriter(maxlen,
vocab_file,
modes=["train"],
task='cls',
check_exist=False)
load = TFLoader(maxlen, batch_size, task='cls', epoch=3)
# init_weights = writer.get_init_weight(word2vec,
# vocab_size,
# embedding_dims)
model = TextCNN.TextCNN(
maxlen,
vocab_size,
embedding_dims,
class_num,
# init_weights,
def predict(self, inputs, is_training=False):
output = self(inputs, is_training=is_training)
return output
model = BERT_NER(param)
model.build(input_shape=(3, param.batch_size, param.maxlen))
model.summary()
# 写入数据 通过check_exist=True参数控制仅在第一次调用时写入
writer = TFWriter(param.maxlen,
vocab_file,
modes=["valid"],
task='cls',
check_exist=True)
load = TFLoader(param.maxlen, param.batch_size, task='cls')
# Metrics
f1score = Metric.SparseF1Score(average="macro")
precsionscore = Metric.SparsePrecisionScore(average="macro")
recallscore = Metric.SparseRecallScore(average="macro")
accuarcyscore = Metric.SparseAccuracy()
# 保存模型
checkpoint = tf.train.Checkpoint(model=model)
checkpoint.restore(tf.train.latest_checkpoint('./save'))
# For train model
output = tf.math.softmax(pre, axis=-1)
return output
def predict(self, inputs, is_training=False):
output = self(inputs, is_training=is_training)
return output
model = BERT_NER(param)
model.build(input_shape=(3, param.batch_size, param.maxlen))
model.summary()
# 写入数据 通过check_exist=True参数控制仅在第一次调用时写入
writer = TFWriter(param.maxlen, vocab_file, modes=["valid"], check_exist=False)
ner_load = TFLoader(param.maxlen, param.batch_size, epoch=3)
# Metrics
f1score = Metric.SparseF1Score(average="macro")
precsionscore = Metric.SparsePrecisionScore(average="macro")
recallscore = Metric.SparseRecallScore(average="macro")
accuarcyscore = Metric.SparseAccuracy()
# 保存模型
checkpoint = tf.train.Checkpoint(model=model)
checkpoint.restore(tf.train.latest_checkpoint('./save'))
# For test model
Batch = 0
predicts = []
from nlpgnn.datas.checkpoint import LoadCheckpoint
from nlpgnn.datas.dataloader import TFWriter
from nlpgnn.tools import bert_init_weights_from_checkpoint
import numpy
# 载入参数
load_check = LoadCheckpoint(language='en', cased=False)
param, vocab_file, model_path = load_check.load_bert_param()
# 定制参数
train_file = "data/R8/r8-train-all-terms.txt"
test_file = "data/R8/r8-test-all-terms.txt"
param.batch_size = 1
param.maxlen = 512
writer = TFWriter(param.maxlen, vocab_file, modes=None)
def target2index(input_file):
label2index = {}
node2index = {"<UNK>": 0}
with open(input_file) as rf:
for line in rf:
line = line.strip().split('\t')
sentences, label = line[1], line[0]
word_set = set(sentences.split(' '))
if label not in label2index:
label2index[label] = len(label2index)
for w in word_set:
if w not in node2index:
node2index[w] = len(node2index)