def ifd_tag_person(txt, p):
i = IFD_Tagset(k="PERSON",
c="person",
g=p.gender,
x=txt,
s=p.name,
t="person_" + p.gender + "_" + p.case)
return str(i)
def ifd_tag(kind, txt, m):
i = IFD_Tagset(k=TOK.descr[kind],
c=m.ordfl,
t=m.ordfl,
f=m.fl,
x=txt,
s=m.stofn,
b=m.beyging)
return str(i)
def tag_stream(sentence_stream):
""" Generator for tag stream from a token stream """
for sent in sentence_stream:
if not sent:
continue
# For each sentence, start and end with empty strings
for i in range(n - 1):
yield ""
for t in sent:
tag = None
# Skip punctuation
if t.get("k", TOK.WORD) != TOK.PUNCTUATION:
canonicalize_token(t)
tag = str(IFD_Tagset(t))
if tag:
self.lemma_cnt[t["x"]][tag] += 1
if tag:
yield tag
for i in range(n - 1):
yield ""
def tag_stream(sentence_stream: Iterable[Iterable[TokenDict]]) -> Iterator[str]:
""" Generator for tag stream from a token stream """
for sent in sentence_stream:
if not sent:
continue
# For each sentence, start and end with empty strings
for _ in range(n - 1):
yield ""
for t in sent:
tag = None
# Skip punctuation
if t.get("k", TOK.WORD) != TOK.PUNCTUATION:
ct = canonicalize_token(t)
tag = str(IFD_Tagset(ct))
if tag:
self.lemma_cnt[ct.get("x", "")][tag] += 1
if tag:
yield tag
for _ in range(n - 1):
yield ""
def ifd_taglist_entity(txt):
i = IFD_Tagset(c="entity", x=txt)
return [(str(i), 1.0)]
def ib_lemmatize(self, g_tokens: TokenList) -> TokLem:
tokens = [t.txt for t in g_tokens]
sent = " ".join(tokens)
if self.ib is None:
raise ValueError("Lemmatizer needs to be instantiated with use_icebert.")
ifds = self.ib.predict_to_idf(sent, device=self.device)
lemmas = []
for idx, tok in enumerate(g_tokens):
cands = tok.val
if isinstance(cands, int) or isinstance(cands, float):
# Number
lemmas.append(tok.txt)
continue
if cands and len(cands) > 1 and (isinstance(cands[0], int) or isinstance(cands[0], float)):
# Punctuation
lemmas.append(tok.txt)
continue
if not cands:
lemmas.append(tok.txt)
continue
lemm_cands = set(c.stofn for c in cands if hasattr(c, "stofn"))
if len(lemm_cands) == 1:
# Only one candidate, we use that one
lemmas.append(lemm_cands.pop())
continue
found = False
for c in cands:
if hasattr(c, "name"):
lemmas.append(c.name)
found = True
break
if isinstance(c[0], int):
lemmas.append(tok.txt)
found = True
break
try:
ifd = IFD_Tagset(
k=tok.kind,
c=c.ordfl,
t=c.ordfl,
f=c.fl,
txt=tok.txt,
s=c.stofn,
b=c.beyging,
)
except: # noqa
lemmas.append(tok.txt)
found = True
break
try:
str_ifd = str(ifd)
except TypeError:
# Some oddity in ifdtagger
str_ifd = ""
if str_ifd == ifds[idx]:
lemmas.append(c.stofn)
found = True
break
if not found:
lemmas.append(tok.txt)
return lemmas, tokens