def process(self, message: Message, **kwargs) -> None:
intent = {"name": None, "confidence": 0.}
intent_ranking = []
tokenizer = load_pretrained_tokenizer(self.pre_path)
decode_pipeline = TrainingPipeLine(device=self.device,
int2idx=self.int2idx,
idx2int=self.idx2int)
if message.text.strip():
score, label = decode_pipeline.decode(model=self.model,
tokenizer=tokenizer,
max_len=self.max_seq_len,
text=message.text,
ranks=INTENT_RANKING_LENGTH)
intent = {"name": label[0], "confidence": score[0]}
intent_ranking = [{
"name": x,
"confidence": y
} for x, y in zip(label[1:], score[1:])]
message.set("intent", intent, add_to_output=True)
message.set("intent_ranking", intent_ranking, add_to_output=True)
def process(self, message: Message, **kwargs: Any):
"""pass"""
intent = {"name": None, "confidence": 0.}
intent_ranking = []
if self.graph is None:
logger.error("No model loaded")
else:
feature = {
"x": message.get("text_features").reshape(1, -1),
"y": np.stack([self.encoded_intents_bag for _ in range(1)]),
"intent": np.array([0])
}
sim, a, b = self.session.run(
[
self.graph.get_tensor_by_name("sim:0"),
self.graph.get_tensor_by_name("text_embed:0"),
self.graph.get_tensor_by_name("intent_embed:0")
],
feed_dict={
self.graph.get_tensor_by_name("text_in:0"): feature["x"],
self.graph.get_tensor_by_name("intent_in:0"): feature["y"],
self.graph.get_tensor_by_name("label_in:0"):
feature["intent"],
self.graph.get_tensor_by_name("dropout:0"): 1.
})
if self.component_config["similarity_type"] == "cosine":
sim[sim < 0.] = 0.
elif self.component_config["similarity_type"] == "inner":
sim = np.exp(sim) / np.sum(sim)
sim = sim.flatten()
intent_ids = np.argsort(sim)[::-1]
intent = {
"name": self.idx2int[intent_ids[0]],
"confidence": sim[intent_ids[0]].tolist()
}
intent_ranking = [{
"name": self.idx2int[intent_ids[idx]],
"confidence": sim[idx].tolist()
} for idx in intent_ids[:INTENT_RANKING_LENGTH]]
message.set("intent", intent, add_to_output=True)
message.set("intent_ranking", intent_ranking, add_to_output=True)
def process(self, message: Message, **kwargs) -> None:
"""prediction"""
if self.vec is None:
logger.error(
"No model loade, and count vectors features pipeline ignored")
return
msg = self._get_message_text(message)
fea = self.vec.transform([msg]).toarray().squeeze()
message.set("text_features",
self._combine_with_existing_text_features(message, fea))
def process(self, message: Message, **kwargs: Any):
"""pass"""
if self.sess is None:
logger.error(
"`BiLSTM-CRF projection model` not trained correctly,"
"components will pass and pipeline procedure continue")
return
res = []
_embedding = self.component_config['embedding']
r_text = message.text
token = [x.text for x in message.get("tokens", [])]
normalizer = self.normalizer
lower = self.component_config["lower_case"]
tag_schema = self.component_config["tag_schema"]
text, words, tags = Processor()._convert_exam(text=r_text,
words=token,
ent_offsets=[],
tag_schema=tag_schema,
normalizer=normalizer,
lower=lower)
if _embedding == "embedding":
res, confidence = self._decode_embedding_entities(
text, words, tags)
elif _embedding == "bert":
res, confidence = self._decode_bert_entities(text, words, tags)
else:
return
# TODO: add confidence to result
res = tagNormalizer(res).run()
extracted = self.add_extractor_name(
pred_result_to_json(r_text, res, confidence))
message.set("entities",
message.get("entities", []) + extracted,
add_to_output=True)
def process(self, message: Message, **kwargs) -> None:
intent = {"name": None, "confidence": 0.}
intent_ranking = []
tokenizer = albert.load_pretrained_tokenizer(self.pre_path)
packer = albert.DataIterPack(message = None,
tokenizer = tokenizer,
max_seq_len = self.max_seq_len,
model = self.model,
int2idx = self.int2idx,
idx2int = self.idx2int,
device = self.device)
if message.text.strip():
score, label = packer.decode(message.text, INTENT_RANKING_LENGTH)
intent = {"name": label[0], "confidence": score[0]}
intent_ranking = [{"name": x, "confidence": y} for x, y in zip(label[1:], score[1:])]
message.set("intent", intent, add_to_output = True)
message.set("intent_ranking", intent_ranking, add_to_output = True)
def process(self, message: Message, **kwargs: Any) -> None:
_intent = {"name": None, "confidence": 0.}
if message.text.strip():
intent = self.parser(message.text)
if intent:
message.set("intent", intent, add_to_output = True)
elif message.get("intent"):
pass
else:
message.set("intent", _intent, add_to_output = True)
else:
message.set("intent", _intent, add_to_output = True)
message.set("intent_ranking", [], add_to_output = True)