def diapivot_annotate(
out: Output = Output(
"<token>:hist.diapivot",
description="SALDO IDs corresponding to lemgrams"),
lemgram: Annotation = Annotation("<token>:saldo.lemgram"),
model: Model = Model("hist/diapivot.pickle")):
"""Annotate each lemgram with its corresponding saldo_id according to model.
Args:
out (str, optional): Resulting annotation file.
Defaults to Output("<token>:hist.diapivot", description="SALDO IDs corresponding to lemgrams").
lemgram (str, optional): Existing lemgram annotation. Defaults to Annotation("<token>:saldo.lemgram").
model (str, optional): Crosslink model. Defaults to Model("hist/diapivot.pickle").
"""
lexicon = PivotLexicon(model)
lemgram_annotation = list(lemgram.read())
out_annotation = []
for lemgrams in lemgram_annotation:
saldo_ids = []
for lemgram in lemgrams.split(util.DELIM):
s_i = lexicon.get_exactMatch(lemgram)
if s_i:
saldo_ids += [s_i]
out_annotation.append(util.AFFIX + util.DELIM.join(set(saldo_ids)) +
util.AFFIX if saldo_ids else util.AFFIX)
out.write(out_annotation)
def uppercase(
word: Annotation = Annotation("<token:word>"),
out: Output = Output("<token>:uppercase.upper"),
# some_config_variable: str = Config("uppercase.some_setting")
):
"""Convert to uppercase."""
out.write([val.upper() for val in word.read()])
def lix(text: Annotation = Annotation("<text>"),
sentence: Annotation = Annotation("<sentence>"),
word: Annotation = Annotation("<token:word>"),
pos: Annotation = Annotation("<token:pos>"),
out: Output = Output("<text>:readability.lix",
description="LIX values for text chunks"),
skip_pos: List[str] = ["MAD", "MID", "PAD"],
fmt: str = "%.2f"):
"""Create LIX annotation for text."""
# Read annotation files and get parent_children relations
text_children, _orphans = text.get_children(sentence)
word_pos = list(word.read_attributes((word, pos)))
sentence_children, _orphans = sentence.get_children(word)
sentence_children = list(sentence_children)
# Calculate LIX for every text element
lix_annotation = []
for text in text_children:
in_sentences = []
for sentence_index in text:
s = sentence_children[sentence_index]
in_sentences.append(
list(
actual_words([word_pos[token_index] for token_index in s],
skip_pos)))
lix_annotation.append(fmt % lix_calc(in_sentences))
out.write(lix_annotation)
def translate_tag(out: Output, tag: Annotation, mapping: dict = {}):
"""Convert part-of-speech tags, specified by the mapping.
Example mappings: parole_to_suc, suc_to_simple, ...
"""
if isinstance(mapping, str):
mapping = util.tagsets.mappings[mapping]
out.write((mapping.get(t, t) for t in tag.read()))
def find_replace_regex(chunk: Annotation,
out: Output,
find: str = "",
sub: str = ""):
"""Do find and replace in values of annotation using a regular expressions.
N.B: When writing regular expressions in YAML they should be enclosed in single quotes.
"""
out.write((re.sub(find, sub, val) for val in chunk.read()))
def concat2(out: Output,
annotations: List[Annotation] = [Annotation],
separator: str = ""):
"""Concatenate two or more annotations, with an optional separator."""
annotations = [list(a.read()) for a in annotations]
out.write([
separator.join([a[n] for a in annotations])
for (n, _) in enumerate(annotations[0])
])
def msdtag(out: Output = Output(
"<token>:hunpos.msd",
cls="token:msd",
description="Part-of-speeches with morphological descriptions"),
word: Annotation = Annotation("<token:word>"),
sentence: Annotation = Annotation("<sentence>"),
binary: Binary = Binary("[hunpos.binary]"),
model: Model = Model("[hunpos.model]"),
morphtable: Optional[Model] = Model("[hunpos.morphtable]"),
patterns: Optional[Model] = Model("[hunpos.patterns]"),
tag_mapping=None,
encoding: str = util.UTF8):
"""POS/MSD tag using the Hunpos tagger."""
if isinstance(tag_mapping, str) and tag_mapping:
tag_mapping = util.tagsets.mappings[tag_mapping]
elif tag_mapping is None or tag_mapping == "":
tag_mapping = {}
pattern_list = []
if patterns:
with open(patterns.path, encoding="utf-8") as pat:
for line in pat:
if line.strip() and not line.startswith("#"):
name, pattern, tags = line.strip().split("\t", 2)
pattern_list.append(
(name, re.compile("^%s$" % pattern), tags))
def replace_word(w):
"""Replace word with alias if word matches a regex pattern."""
for p in pattern_list:
if re.match(p[1], w):
return "[[%s]]" % p[0]
return w
sentences, _orphans = sentence.get_children(word)
token_word = list(word.read())
stdin = SENT_SEP.join(
TOK_SEP.join(
replace_word(token_word[token_index]) for token_index in sent)
for sent in sentences)
args = [model.path]
if morphtable:
args.extend(["-m", morphtable.path])
stdout, _ = util.system.call_binary(binary, args, stdin, encoding=encoding)
out_annotation = word.create_empty_attribute()
for sent, tagged_sent in zip(sentences, stdout.strip().split(SENT_SEP)):
for token_index, tagged_token in zip(
sent,
tagged_sent.strip().split(TOK_SEP)):
tag = tagged_token.strip().split(TAG_SEP)[TAG_COLUMN]
tag = tag_mapping.get(tag, tag)
out_annotation[token_index] = tag
out.write(out_annotation)
def override(chunk: Annotation, repl: Annotation, out: Output):
"""Replace values in 'chunk' with non empty values from 'repl'."""
def empty(val):
if not val:
return True
return val == "|"
repl = list(repl.read())
out.write((repl[n] if not empty(repl[n]) else val
for (n, val) in enumerate(chunk.read())))
def struct_to_token(
attr: Annotation = Annotation("{struct}:{attr}"),
token: Annotation = Annotation("<token>"),
out: Output = Output("<token>:misc.from_struct_{struct}_{attr}")):
"""Convert an attribute on a structural annotation into a token attribute."""
token_parents = token.get_parents(attr)
attr_values = list(attr.read())
out_values = [
attr_values[p] if p is not None else "" for p in token_parents
]
out.write(out_values)
def select(out: Output,
annotation: Annotation,
index: Optional[int] = 0,
separator: Optional[str] = " "):
"""Select a specific index from the values of an annotation.
The given annotation values are separated by 'separator',
by default whitespace, with at least index + 1 elements.
"""
if isinstance(index, str):
index = int(index)
out.write(value.split(separator)[index] for value in annotation.read())
def upostag(out: Output = Output("<token>:misc.upos",
cls="token:upos",
description="Part-of-speeches in UD"),
pos: Annotation = Annotation("<token:pos>")):
"""Convert SUC POS tags to UPOS."""
pos_tags = pos.read()
out_annotation = []
for tag in pos_tags:
out_annotation.append(util.tagsets.pos_to_upos(tag, "swe", "SUC"))
out.write(out_annotation)
def concat(out: Output,
left: Annotation,
right: Annotation,
separator: str = "",
merge_twins: bool = False):
"""Concatenate values from two annotations, with an optional separator.
If merge_twins is set to True, no concatenation will be done on identical values.
"""
b = list(right.read())
out.write((f"{val_a}{separator}{b[n]}"
if not (merge_twins and val_a == b[n]) else val_a
for (n, val_a) in enumerate(left.read())))
def annotate(
sense: Annotation = Annotation("<token>:saldo.sense"),
out_scores: Output = Output("<token>:sensaldo.sentiment_score",
description="SenSALDO sentiment score"),
out_labels: Output = Output("<token>:sensaldo.sentiment_label",
description="SenSALDO sentiment label"),
model: Model = Model("[sensaldo.model]"),
lexicon=None):
"""Assign sentiment values to tokens based on their sense annotation.
When more than one sense is possible, calulate a weighted mean.
- sense: existing annotation with saldoIDs.
- out_scores, out_labels: resulting annotation file.
- model: pickled lexicon with saldoIDs as keys.
- lexicon: this argument cannot be set from the command line,
but is used in the catapult. This argument must be last.
"""
if not lexicon:
lexicon = util.PickledLexicon(model.path)
# Otherwise use pre-loaded lexicon (from catapult)
sense = sense.read()
result_scores = []
result_labels = []
for token in sense:
# Get set of senses for each token and sort them according to their probabilities
token_senses = [
tuple(s.rsplit(util.SCORESEP, 1)) if util.SCORESEP in s else
(s, -1.0) for s in token.split(util.DELIM) if s
]
token_senses.sort(key=lambda x: float(x[1]), reverse=True)
# Lookup the sentiment score for the most probable sense and assign a sentiment label
if token_senses:
best_sense = token_senses[0][0]
score = lexicon.lookup(best_sense, None)
else:
score = None
if score:
result_scores.append(score)
result_labels.append(SENTIMENT_LABLES.get(int(score)))
else:
result_scores.append(None)
result_labels.append(None)
out_scores.write(result_scores)
out_labels.write(result_labels)
def contextual(out: Output = Output("{chunk}:geo.geo_context", description="Geographical places with coordinates"),
chunk: Annotation = Annotation("{chunk}"),
context: Annotation = Annotation("[geo.context_chunk]"),
ne_type: Annotation = Annotation("swener.ne:swener.type"),
ne_subtype: Annotation = Annotation("swener.ne:swener.subtype"),
ne_name: Annotation = Annotation("swener.ne:swener.name"),
model: Model = Model("[geo.model]"),
method: str = "populous",
language: list = []):
"""Annotate chunks with location data, based on locations contained within the text.
context = text chunk to use for disambiguating places (when applicable).
chunk = text chunk to which the annotation will be added.
"""
model = load_model(model, language=language)
ne_type_annotation = list(ne_type.read())
ne_subtype_annotation = list(ne_subtype.read())
ne_name_annotation = list(ne_name.read())
children_context_chunk, _orphans = context.get_children(chunk)
children_chunk_ne, _orphans = chunk.get_children(ne_type)
out_annotation = chunk.create_empty_attribute()
for chunks in children_context_chunk:
all_locations = [] # TODO: Maybe not needed for anything?
context_locations = []
chunk_locations = defaultdict(list)
for ch in chunks:
for n in children_chunk_ne[ch]:
if ne_type_annotation[n] == "LOC" and "PPL" in ne_subtype_annotation[n]:
location_text = ne_name_annotation[n].replace("\n", " ").replace(" ", " ")
location_data = model.get(location_text.lower())
if location_data:
all_locations.append((location_text, list(location_data)))
context_locations.append((location_text, list(location_data)))
chunk_locations[ch].append((location_text, list(location_data)))
else:
pass
# log.info("No location found for %s" % ne_name_annotation[n].replace("%", "%%"))
chunk_locations = most_populous(chunk_locations)
for c in chunks:
out_annotation[c] = _format_location(chunk_locations.get(c, ()))
out.write(out_annotation)
def number_relative(out: Output = Output("{annotation}:misc.number_rel_{parent}"),
parent: Annotation = Annotation("{parent}"),
child: Annotation = Annotation("{annotation}"),
prefix: str = "",
zfill: bool = False,
start: int = START_DEFAULT):
"""Number chunks by their relative position within a parent."""
parent_children, _orphans = parent.get_children(child)
out.write(("{prefix}{nr:0{length}d}".format(prefix=prefix,
length=len(str(len(parent) - 1 + start))
if zfill else 0,
nr=cnr)
for parent in parent_children
for cnr, _index in enumerate(parent, start)))
def ufeatstag(out: Output = Output(
"<token>:misc.ufeats",
cls="token:ufeats",
description="Universal morphological features"),
pos: Annotation = Annotation("<token:pos>"),
msd: Annotation = Annotation("<token:msd>")):
"""Convert SUC MSD tags to universal features."""
pos_tags = pos.read()
msd_tags = msd.read()
out_annotation = []
for pos_tag, msd_tag in zip(pos_tags, msd_tags):
feats = util.tagsets.suc_to_feats(pos_tag, msd_tag)
out_annotation.append(util.cwbset(feats))
out.write(out_annotation)
def replace_list(chunk: Annotation,
out: Output,
find: str = "",
sub: str = ""):
"""Find and replace annotations.
Find string must match whole annotation.
find and sub are whitespace separated lists of words to replace and their replacement.
"""
find = find.split()
sub = sub.split()
if len(find) != len(sub):
raise util.SparvErrorMessage(
"Find and sub must have the same number of words.")
translate = dict((f, s) for (f, s) in zip(find, sub))
out.write((translate.get(val, val) for val in chunk.read()))
def process_output(word: Annotation, out: Output, stdout, in_sentences,
saldo_annotation, prob_format, default_prob):
"""Parse WSD output and write annotation."""
out_annotation = word.create_empty_attribute()
# Split output into sentences
out_sentences = stdout.strip()
out_sentences = out_sentences.split("\t".join(
["_", "_", "_", "_", SENT_SEP, "_", "_"]))
out_sentences = [i for i in out_sentences if i]
# Split output into tokens
for out_sent, in_sent in zip(out_sentences, in_sentences):
out_tokens = [t for t in out_sent.split("\n") if t]
for (out_tok, in_tok) in zip(out_tokens, in_sent):
out_prob = out_tok.split("\t")[6]
out_prob = [i for i in out_prob.split("|") if i != "_"]
out_meanings = [
i for i in out_tok.split("\t")[5].split("|") if i != "_"
]
saldo = [
i for i in saldo_annotation[in_tok].strip(util.AFFIX).split(
util.DELIM) if i
]
new_saldo = []
if out_prob:
for meaning in saldo:
if meaning in out_meanings:
i = out_meanings.index(meaning)
new_saldo.append((meaning, float(out_prob[i])))
else:
new_saldo.append((meaning, default_prob))
else:
new_saldo = [(meaning, default_prob) for meaning in saldo]
# Sort by probability
new_saldo.sort(key=lambda x: (-x[1], x[0]))
# Format probability according to prob_format
new_saldo = [
saldo + prob_format % prob if prob_format else saldo
for saldo, prob in new_saldo
]
out_annotation[in_tok] = util.cwbset(new_saldo)
out.write(out_annotation)
def ids(doc: Document = Document(),
annotation: Annotation = Annotation("{annotation}"),
out: Output = Output("{annotation}:misc.id",
description="Unique ID for {annotation}"),
docid: AnnotationData = AnnotationData("<docid>"),
prefix: str = ""):
"""Create unique IDs for every span of an existing annotation."""
docid = docid.read()
prefix = prefix + docid
ann = list(annotation.read())
out_annotation = []
# Use doc name and annotation name as seed for the IDs
_reset_id("{}/{}".format(doc, annotation), len(ann))
for _ in ann:
new_id = _make_id(prefix, out_annotation)
out_annotation.append(new_id)
out.write(out_annotation)
def nominal_ratio(text: Annotation = Annotation("<text>"),
pos: Annotation = Annotation("<token:pos>"),
out: Output = Output(
"<text>:readability.nk",
description="Nominal ratios for text chunks"),
noun_pos: List[str] = ["NN", "PP", "PC"],
verb_pos: List[str] = ["PN", "AB", "VB"],
fmt: str = "%.2f"):
"""Create nominal ratio annotation for text."""
text_children, _orphans = text.get_children(pos)
pos_annotation = list(pos.read())
# Calculate OVIX for every text element
nk_annotation = []
for text in text_children:
in_pos = [pos_annotation[token_index] for token_index in text]
nk_annotation.append(fmt %
nominal_ratio_calc(in_pos, noun_pos, verb_pos))
out.write(nk_annotation)
def ovix(text: Annotation = Annotation("<text>"),
word: Annotation = Annotation("<token:word>"),
pos: Annotation = Annotation("<token:pos>"),
out: Output = Output("<text>:readability.ovix",
description="OVIX values for text chunks"),
skip_pos: List[str] = ["MAD", "MID", "PAD"],
fmt: str = "%.2f"):
"""Create OVIX annotation for text."""
text_children, _orphans = text.get_children(word)
word_pos = list(word.read_attributes((word, pos)))
# Calculate OVIX for every text element
ovix_annotation = []
for text in text_children:
in_words = list(
actual_words([word_pos[token_index] for token_index in text],
skip_pos))
ovix_annotation.append(fmt % ovix_calc(in_words))
out.write(ovix_annotation)
def metadata(out: Output = Output("{chunk}:geo.geo_metadata", description="Geographical places with coordinates"),
chunk: Annotation = Annotation("{chunk}"),
source: Annotation = Annotation("[geo.metadata_source]"),
model: Model = Model("[geo.model]"),
method: str = "populous",
language: list = []):
"""Get location data based on metadata containing location names."""
geomodel = load_model(model, language=language)
same_target_source = chunk.split()[0] == source.split()[0]
chunk_annotation = list(chunk.read())
source_annotation = list(source.read())
# If location source and target chunk are not the same, we need
# to find the parent/child relations between them.
if not same_target_source:
target_source_parents = list(source.get_parents(chunk))
chunk_locations = {}
for i, _ in enumerate(chunk_annotation):
if same_target_source:
location_source = source_annotation[i]
else:
location_source = source_annotation[target_source_parents[i]] if target_source_parents[
i] is not None else None
if location_source:
location_data = geomodel.get(location_source.strip().lower())
if location_data:
chunk_locations[i] = [(location_source, list(location_data))]
else:
chunk_locations[i] = []
chunk_locations = most_populous(chunk_locations)
out_annotation = chunk.create_empty_attribute()
for c in chunk_locations:
out_annotation[c] = _format_location(chunk_locations.get(c, ()))
out.write(out_annotation)
def _read_chunks_and_write_new_ordering(out: Output, chunk: Annotation, order, prefix="", zfill=False,
start=START_DEFAULT):
"""Common function called by other numbering functions."""
new_order = defaultdict(list)
in_annotation = list(chunk.read())
for i, val in enumerate(in_annotation):
val = order(i, val)
new_order[val].append(i)
out_annotation = chunk.create_empty_attribute()
nr_digits = len(str(len(new_order) - 1 + start))
for nr, key in enumerate(sorted(new_order), start):
for index in new_order[key]:
out_annotation[index] = "{prefix}{nr:0{length}d}".format(prefix=prefix,
length=nr_digits if zfill else 0,
nr=nr)
out.write(out_annotation)
def text_spans(text: Text = Text(),
chunk: Annotation = Annotation("<token>"),
out: Output = Output("<token>:misc.word", cls="token:word"),
keep_formatting_chars: Optional[bool] = Config(
"misc.keep_formatting_chars")):
"""Add the text content for each edge as a new annotation."""
corpus_text = text.read()
if isinstance(chunk, (str, Annotation)):
chunk = chunk.read_spans()
out_annotation = []
for span in chunk:
token = corpus_text[span[0]:span[1]]
if not keep_formatting_chars:
new_token = util.remove_formatting_characters(token)
# If this token consists entirely of formatting characters, don't remove them. Empty tokens are bad!
if new_token:
token = new_token
out_annotation.append(token)
if out:
out.write(out_annotation)
else:
return out_annotation
def text_headtail(text: Text = Text(),
chunk: Annotation = Annotation("<token>"),
out_head: Output = Output("<token>:misc.head"),
out_tail: Output = Output("<token>:misc.tail")):
"""Extract "head" and "tail" whitespace characters for tokens."""
def escape(t):
"""Escape whitespace characters."""
return t.replace(" ", "\\s").replace("\n", "\\n").replace("\t", "\\t")
out_head_annotation = chunk.create_empty_attribute()
out_tail_annotation = chunk.create_empty_attribute()
head_text = None
corpus_text = text.read()
chunk = list(chunk.read())
for i, span in enumerate(chunk):
if head_text:
out_head_annotation[i] = escape(head_text)
head_text = None
if i < len(chunk) - 1:
tail_start = span[1][0]
tail_end = chunk[i + 1][0][0]
tail_text = corpus_text[tail_start:tail_end]
try:
n_pos = tail_text.rindex("\n")
except ValueError:
n_pos = None
if n_pos is not None and n_pos + 1 < len(tail_text):
head_text = tail_text[n_pos + 1:]
tail_text = tail_text[:n_pos + 1]
if tail_text:
out_tail_annotation[i] = escape(tail_text)
out_head.write(out_head_annotation)
out_tail.write(out_tail_annotation)
def annotate(
lang: Language = Language(),
model: Model = Model("[treetagger.model]"),
tt_binary: Binary = Binary("[treetagger.binary]"),
out_upos: Output = Output("<token>:treetagger.upos",
cls="token:upos",
description="Part-of-speeches in UD"),
out_pos: Output = Output(
"<token>:treetagger.pos",
cls="token:pos",
description="Part-of-speeches from TreeTagger"),
out_baseform: Output = Output("<token>:treetagger.baseform",
description="Baseforms from TreeTagger"),
word: Annotation = Annotation("<token:word>"),
sentence: Annotation = Annotation("<sentence>"),
encoding: str = util.UTF8):
"""POS/MSD tag and lemmatize using TreeTagger."""
sentences, _orphans = sentence.get_children(word)
word_annotation = list(word.read())
stdin = SENT_SEP.join(
TOK_SEP.join(word_annotation[token_index] for token_index in sent)
for sent in sentences)
args = ["-token", "-lemma", "-no-unknown", "-eos-tag", "<eos>", model.path]
stdout, stderr = util.system.call_binary(tt_binary,
args,
stdin,
encoding=encoding)
log.debug("Message from TreeTagger:\n%s", stderr)
# Write pos and upos annotations.
out_upos_annotation = word.create_empty_attribute()
out_pos_annotation = word.create_empty_attribute()
for sent, tagged_sent in zip(sentences, stdout.strip().split(SENT_SEP)):
for token_id, tagged_token in zip(sent,
tagged_sent.strip().split(TOK_SEP)):
tag = tagged_token.strip().split(TAG_SEP)[TAG_COLUMN]
out_pos_annotation[token_id] = tag
out_upos_annotation[token_id] = util.tagsets.pos_to_upos(
tag, lang, TAG_SETS.get(lang))
out_pos.write(out_pos_annotation)
out_upos.write(out_upos_annotation)
# Write lemma annotations.
out_lemma_annotation = word.create_empty_attribute()
for sent, tagged_sent in zip(sentences, stdout.strip().split(SENT_SEP)):
for token_id, tagged_token in zip(sent,
tagged_sent.strip().split(TOK_SEP)):
lem = tagged_token.strip().split(TAG_SEP)[LEM_COLUMN]
out_lemma_annotation[token_id] = lem
out_baseform.write(out_lemma_annotation)
def annotate(
maltjar: Binary = Binary("[malt.jar]"),
model: Model = Model("[malt.model]"),
out_dephead: Output = Output(
"<token>:malt.dephead",
cls="token:dephead",
description="Positions of the dependency heads"),
out_dephead_ref: Output = Output(
"<token>:malt.dephead_ref",
cls="token:dephead_ref",
description="Sentence-relative positions of the dependency heads"),
out_deprel: Output = Output(
"<token>:malt.deprel",
cls="token:deprel",
description="Dependency relations to the head"),
word: Annotation = Annotation("<token:word>"),
pos: Annotation = Annotation("<token:pos>"),
msd: Annotation = Annotation("<token:msd>"),
ref: Annotation = Annotation("<token>:misc.number_rel_<sentence>"),
sentence: Annotation = Annotation("<sentence>"),
token: Annotation = Annotation("<token>"),
encoding: str = util.UTF8,
process_dict=None):
"""
Run the malt parser, in an already started process defined in process_dict, or start a new process (default).
The process_dict argument should never be set from the command line.
"""
if process_dict is None:
process = maltstart(maltjar, model, encoding)
else:
process = process_dict["process"]
# If process seems dead, spawn a new
if process.stdin.closed or process.stdout.closed or process.poll():
util.system.kill_process(process)
process = maltstart(maltjar,
model,
encoding,
send_empty_sentence=True)
process_dict["process"] = process
sentences, orphans = sentence.get_children(token)
sentences.append(orphans)
word_annotation = list(word.read())
pos_annotation = list(pos.read())
msd_annotation = list(msd.read())
ref_annotation = list(ref.read())
def conll_token(nr, token_index):
form = word_annotation[token_index]
lemma = UNDEF
pos = cpos = pos_annotation[token_index]
feats = re.sub(r"[ ,.]", "|",
msd_annotation[token_index]).replace("+", "/")
return TAG_SEP.join((str(nr), form, lemma, cpos, pos, feats))
stdin = SENT_SEP.join(
TOK_SEP.join(
conll_token(n + 1, token_index)
for n, token_index in enumerate(sent)) for sent in sentences)
if encoding:
stdin = stdin.encode(encoding)
keep_process = len(
stdin) < RESTART_THRESHOLD_LENGTH and process_dict is not None
log.info("Stdin length: %s, keep process: %s", len(stdin), keep_process)
if process_dict is not None:
process_dict["restart"] = not keep_process
if keep_process:
# Chatting with malt: send a SENT_SEP and read correct number of lines
stdin_fd, stdout_fd = process.stdin, process.stdout
stdin_fd.write(stdin + SENT_SEP.encode(util.UTF8))
stdin_fd.flush()
malt_sentences = []
for sent in sentences:
malt_sent = []
for _ in sent:
line = stdout_fd.readline()
if encoding:
line = line.decode(encoding)
malt_sent.append(line)
line = stdout_fd.readline()
assert line == b"\n"
malt_sentences.append(malt_sent)
else:
# Otherwise use communicate which buffers properly
stdout, _ = process.communicate(stdin)
if encoding:
stdout = stdout.decode(encoding)
malt_sentences = (malt_sent.split(TOK_SEP)
for malt_sent in stdout.split(SENT_SEP))
out_dephead_annotation = word.create_empty_attribute()
out_dephead_ref_annotation = out_dephead_annotation.copy()
out_deprel_annotation = out_dephead_annotation.copy()
for (sent, malt_sent) in zip(sentences, malt_sentences):
for (token_index, malt_tok) in zip(sent, malt_sent):
cols = [(None if col == UNDEF else col)
for col in malt_tok.split(TAG_SEP)]
out_deprel_annotation[token_index] = cols[DEPREL_COLUMN]
head = int(cols[HEAD_COLUMN])
out_dephead_annotation[token_index] = str(sent[head -
1]) if head else "-"
out_dephead_ref_annotation[token_index] = str(
ref_annotation[sent[head - 1]]) if head else ""
out_dephead.write(out_dephead_annotation)
out_dephead_ref.write(out_dephead_ref_annotation)
out_deprel.write(out_deprel_annotation)
def annotate_words(out: Output, model: Model, saldoids: Annotation, pos: Annotation, annotate, pos_limit: List[str],
class_set=None, disambiguate=True, connect_ids=False, delimiter=util.DELIM, affix=util.AFFIX,
scoresep=util.SCORESEP, lexicon=None):
"""
Annotate words with blingbring classes (rogetID).
- out_sent: resulting annotation file.
- model: pickled lexicon with saldoIDs as keys.
- saldoids, pos: existing annotation with saldoIDs/parts of speech.
- annotate: annotation function, returns an iterable containing annotations
for one token ID. (annotate_bring() or annotate_swefn())
- pos_limit: parts of speech that will be annotated.
Set to None to annotate all pos.
- class_set: output Bring classes or Roget IDs ("bring", "roget_head",
"roget_subsection", "roget_section" or "roget_class").
Set to None when not annotating blingbring.
- disambiguate: use WSD and use only the most likely saldo ID.
- connect_IDs: for sweFN: paste saldo ID after each sweFN ID.
- delimiter: delimiter character to put between ambiguous results
- affix: optional character to put before and after results to mark a set.
- lexicon: this argument cannot be set from the command line,
but is used in the catapult. This argument must be last.
"""
if not lexicon:
lexicon = util.PickledLexicon(model.path)
# Otherwise use pre-loaded lexicon (from catapult)
sense = saldoids.read()
token_pos = list(pos.read())
out_annotation = pos.create_empty_attribute()
# Check if the saldo IDs are ranked (= word senses have been disambiguated)
wsd = saldoids.split()[1].split(".")[0] == "wsd"
for token_index, token_sense in enumerate(sense):
# Check if part of speech of this token is allowed
if not pos_ok(token_pos, token_index, pos_limit):
saldo_ids = None
out_annotation[token_index] = affix
continue
if wsd and util.SCORESEP in token_sense:
ranked_saldo = token_sense.strip(util.AFFIX).split(util.DELIM) \
if token_sense != util.AFFIX else None
saldo_tuples = [(i.split(util.SCORESEP)[0], i.split(util.SCORESEP)[1]) for i in ranked_saldo]
if not disambiguate:
saldo_ids = [i[0] for i in saldo_tuples]
# Only take the most likely analysis into account.
# Handle wsd with equal probability for several words
else:
saldo_ids = [saldo_tuples[0]]
del saldo_tuples[0]
while saldo_tuples and (saldo_tuples[0][1] == saldo_ids[0][1]):
saldo_ids = [saldo_tuples[0]]
del saldo_tuples[0]
saldo_ids = [i[0] for i in saldo_ids]
else: # No WSD
saldo_ids = token_sense.strip(util.AFFIX).split(util.DELIM) \
if token_sense != util.AFFIX else None
result = annotate(saldo_ids, lexicon, connect_ids, scoresep)
out_annotation[token_index] = util.cwbset(result, delimiter, affix) if result else affix
out.write(out_annotation)
def do_segmentation(text: Text,
out: Output,
segmenter,
chunk: Optional[Annotation] = None,
existing_segments=None,
model: Optional[Model] = None,
token_list: Optional[Model] = None):
"""Segment all chunks (e.g. sentences) into smaller "tokens" (e.g. words), and annotate them as "element" (e.g. w).
Segmentation is done by the given "segmenter"; some segmenters take
an extra argument which is a pickled "model" object.
"""
segmenter_args = []
if model:
if model.path.suffix in ["pickle", "pkl"]:
with open(model, "rb") as M:
model_arg = pickle.load(M, encoding="UTF-8")
else:
model_arg = str(model.path)
segmenter_args.append(model_arg)
assert segmenter in SEGMENTERS, "Available segmenters: %s" % ", ".join(
sorted(SEGMENTERS))
segmenter = SEGMENTERS[segmenter]
segmenter = segmenter(*segmenter_args)
assert hasattr(
segmenter, "span_tokenize"
), "Segmenter needs a 'span_tokenize' method: %r" % segmenter
corpus_text = text.read()
# First we read the chunks and partition the text into spans
# E.g., "one two <s>three four</s> five <s>six</s>"
# ==> ["one two ", "three four", " five ", "six"]
# (but using spans (pairs of anchors) instead of strings)
positions = set()
chunk_spans = chunk.read_spans() if chunk else []
positions = positions.union(
set(pos for span in chunk_spans for pos in span))
positions = sorted({0, len(corpus_text)} | positions)
chunk_spans = list(zip(positions, positions[1:]))
if existing_segments:
segments = list(existing_segments.read_spans())
for n, (chunk_start, chunk_end) in enumerate(chunk_spans[:]):
for segment_start, segment_end in segments:
if segment_end <= chunk_start:
continue
if segment_start >= chunk_end:
break
if chunk_start != segment_start:
chunk_spans.append((chunk_start, segment_start))
chunk_start = segment_end
chunk_spans[n] = (chunk_start, chunk_end)
chunk_spans.sort()
log.info("Reorganized into %d chunks" % len(chunk_spans))
else:
segments = []
# Now we can segment each chunk span into tokens
for start, end in chunk_spans:
for spanstart, spanend in segmenter.span_tokenize(
corpus_text[start:end]):
spanstart += start
spanend += start
if corpus_text[spanstart:spanend].strip():
span = (spanstart, spanend)
segments.append(span)
segments.sort()
out.write(segments)
def _formatter(in_from: Annotation, in_to: Optional[Annotation],
out_from: Output, out_to: Output, informat: str, outformat: str,
splitter: str, regex: str):
"""Take existing dates/times and input formats and convert to specified output format."""
def get_smallest_unit(informat):
smallest_unit = 0 # No date
if "%y" not in informat and "%Y" not in informat:
pass
elif "%b" not in informat and "%B" not in informat and "%m" not in informat:
smallest_unit = 1 # year
elif "%d" not in informat:
smallest_unit = 2 # month
elif "%H" not in informat and "%I" not in informat:
smallest_unit = 3 # day
elif "%M" not in informat:
smallest_unit = 4 # hour
elif "%S" not in informat:
smallest_unit = 5 # minute
else:
smallest_unit = 6 # second
return smallest_unit
def get_date_length(informat):
parts = informat.split("%")
length = len(
parts[0]) # First value is either blank or not part of date
lengths = {
"Y": 4,
"3Y": 3,
"y": 2,
"m": 2,
"b": None,
"B": None,
"d": 2,
"H": None,
"I": None,
"M": 2,
"S": 2
}
for part in parts[1:]:
add = lengths.get(part[0], None)
if add:
length += add + len(part[1:])
else:
return None
return length
if not in_to:
in_to = in_from
informat = informat.split("|")
outformat = outformat.split("|")
if splitter:
splitter = splitter
assert len(outformat) == 1 or (len(outformat) == len(informat)), "The number of out-formats must be equal to one " \
"or the number of in-formats."
ifrom = list(in_from.read())
ofrom = in_from.create_empty_attribute()
for index, val in enumerate(ifrom):
val = val.strip()
if not val:
ofrom[index] = None
continue
tries = 0
for inf in informat:
if splitter and splitter in inf:
values = re.findall("%[YybBmdHMS]", inf)
if len(set(values)) < len(values):
vals = val.split(splitter)
inf = inf.split(splitter)
else:
vals = [val]
inf = [inf]
if regex:
temp = []
for v in vals:
matches = re.search(regex, v)
if matches:
temp.append([x for x in matches.groups() if x][0])
if not temp:
# If the regex doesn't match, treat as no date
ofrom[index] = None
continue
vals = temp
tries += 1
try:
fromdates = []
for i, v in enumerate(vals):
if "%3Y" in inf[i]:
datelen = get_date_length(inf[i])
if datelen and not datelen == len(v):
raise ValueError
inf[i] = inf[i].replace("%3Y", "%Y")
v = "0" + v
if "%0m" in inf[i] or "%0d" in inf[i]:
inf[i] = inf[i].replace("%0m",
"%m").replace("%0d", "%d")
datelen = get_date_length(inf[i])
if datelen and not datelen == len(v):
raise ValueError
fromdates.append(datetime.datetime.strptime(v, inf[i]))
if len(fromdates) == 1 or out_to:
ofrom[index] = fromdates[0].strftime(outformat[0] if len(
outformat) == 1 else outformat[tries - 1])
else:
outstrings = [
fromdate.strftime(outformat[0] if len(outformat) ==
1 else outformat[tries - 1])
for fromdate in fromdates
]
ofrom[index] = outstrings[0] + splitter + outstrings[1]
break
except ValueError:
if tries == len(informat):
log.error("Could not parse: %s", str(vals))
raise
continue
out_from.write(ofrom)
del ofrom
if out_to:
ito = list(in_to.read())
oto = in_to.create_empty_attribute()
for index, val in enumerate(ito):
if not val:
oto[index] = None
continue
tries = 0
for inf in informat:
if splitter and splitter in inf:
values = re.findall("%[YybBmdHMS]", inf)
if len(set(values)) < len(values):
vals = val.split(splitter)
inf = inf.split(splitter)
else:
vals = [val]
inf = [inf]
if regex:
temp = []
for v in vals:
matches = re.search(regex, v)
if matches:
temp.append([x for x in matches.groups() if x][0])
if not temp:
# If the regex doesn't match, treat as no date
oto[index] = None
continue
vals = temp
tries += 1
try:
todates = []
for i, v in enumerate(vals):
if "%3Y" in inf[i]:
datelen = get_date_length(inf[i])
if datelen and not datelen == len(v):
raise ValueError
inf[i] = inf[i].replace("%3Y", "%Y")
v = "0" + v
if "%0m" in inf[i] or "%0d" in inf[i]:
inf[i] = inf[i].replace("%0m",
"%m").replace("%0d", "%d")
datelen = get_date_length(inf[i])
if datelen and not datelen == len(v):
raise ValueError
todates.append(datetime.datetime.strptime(v, inf[i]))
smallest_unit = get_smallest_unit(inf[0])
if smallest_unit == 1:
add = relativedelta(years=1)
elif smallest_unit == 2:
add = relativedelta(months=1)
elif smallest_unit == 3:
add = relativedelta(days=1)
elif smallest_unit == 4:
add = relativedelta(hours=1)
elif smallest_unit == 5:
add = relativedelta(minutes=1)
elif smallest_unit == 6:
add = relativedelta(seconds=1)
todates = [
todate + add - relativedelta(seconds=1)
for todate in todates
]
oto[index] = todates[-1].strftime(outformat[0] if len(
outformat) == 1 else outformat[tries - 1])
break
except ValueError:
if tries == len(informat):
log.error("Could not parse: %s", str(vals))
raise
continue
out_to.write(oto)