def load_vocab(vocab_dir):
"""load vocabs"""
word_vocab = Vocab.from_json(os.path.join(vocab_dir, "word_vocab.json"))
rel_vocab = Vocab.from_json(os.path.join(vocab_dir, "rel_vocab.json"))
feat_vocab_path = os.path.join(vocab_dir, "feat_vocab.json")
if os.path.exists(feat_vocab_path):
feat_vocab = Vocab.from_json(os.path.join(feat_vocab_path))
else:
feat_vocab = None
return word_vocab, feat_vocab, rel_vocab
def test_json(self):
token_to_idx = {'一万七千多': 1, '一万七千余': 2, '一万万': 3}
vocab = Vocab(
counter=self.counter, unk_token='[UNK]', token_to_idx=token_to_idx)
json_str = vocab.to_json()
copied_vocab = Vocab.from_json(json_str)
for key, value in copied_vocab.token_to_idx.items():
self.check_output_equal(value, vocab[key])
print(probs)
idx = np.argmax(probs, axis=1)
idx = idx.tolist()
labels = [label_map[i] for i in idx]
results.extend(labels)
return results
if __name__ == "__main__":
# Define predictor to do prediction.
predictor = Predictor(args.model_file, args.params_file, args.device,
args.max_seq_length)
# Firstly pre-processing prediction data and then do predict.
data = [
'非常不错，服务很好，位于市中心区，交通方便，不过价格也高！',
'怀着十分激动的心情放映，可是看着看着发现，在放映完毕后，出现一集米老鼠的动画片',
'作为老的四星酒店，房间依然很整洁，相当不错。机场接机服务很好，可以在车上办理入住手续，节省时间。',
]
vocab = Vocab.from_json(args.vocab_path)
tokenizer = JiebaTokenizer(vocab)
label_map = {0: 'negative', 1: 'positive'}
results = predictor.predict(data,
tokenizer,
label_map,
batch_size=args.batch_size,
network=args.network)
for idx, text in enumerate(data):
print('Data: {} \t Label: {}'.format(text, results[idx]))
def main():
# Load vocab.
vocab = Vocab.from_json(args.vocab_path)
label_map = {0: 'negative', 1: 'positive'}
# Constructs the newtork.
network = args.network.lower()
vocab_size = len(vocab)
num_classes = len(label_map)
pad_token_id = vocab.to_indices('[PAD]')
if network == 'bow':
model = BoWModel(vocab_size, num_classes, padding_idx=pad_token_id)
elif network == 'bigru':
model = GRUModel(vocab_size,
num_classes,
direction='bidirect',
padding_idx=pad_token_id)
elif network == 'bilstm':
model = LSTMModel(vocab_size,
num_classes,
direction='bidirect',
padding_idx=pad_token_id)
elif network == 'bilstm_attn':
lstm_hidden_size = 196
attention = SelfInteractiveAttention(hidden_size=2 * lstm_hidden_size)
model = BiLSTMAttentionModel(attention_layer=attention,
vocab_size=vocab_size,
lstm_hidden_size=lstm_hidden_size,
num_classes=num_classes,
padding_idx=pad_token_id)
elif network == 'birnn':
model = RNNModel(vocab_size,
num_classes,
direction='bidirect',
padding_idx=pad_token_id)
elif network == 'cnn':
model = CNNModel(vocab_size, num_classes, padding_idx=pad_token_id)
elif network == 'gru':
model = GRUModel(vocab_size,
num_classes,
direction='forward',
padding_idx=pad_token_id,
pooling_type='max')
elif network == 'lstm':
model = LSTMModel(vocab_size,
num_classes,
direction='forward',
padding_idx=pad_token_id,
pooling_type='max')
elif network == 'rnn':
model = RNNModel(vocab_size,
num_classes,
direction='forward',
padding_idx=pad_token_id,
pooling_type='max')
else:
raise ValueError(
"Unknown network: %s, it must be one of bow, lstm, bilstm, cnn, gru, bigru, rnn, birnn and bilstm_attn."
% network)
# Load model parameters.
state_dict = paddle.load(args.params_path)
model.set_dict(state_dict)
model.eval()
inputs = [paddle.static.InputSpec(shape=[None, None], dtype="int64")]
# Convert to static graph with specific input description
if args.network in [
"lstm", "bilstm", "gru", "bigru", "rnn", "birnn", "bilstm_attn"
]:
inputs.append(paddle.static.InputSpec(shape=[None],
dtype="int64")) # seq_len
model = paddle.jit.to_static(model, input_spec=inputs)
# Save in static graph model.
paddle.jit.save(model, args.output_path)
