def diapivot_annotate( out: Output = Output( "<token>:hist.diapivot", description="SALDO IDs corresponding to lemgrams"), lemgram: Annotation = Annotation("<token>:saldo.lemgram"), model: Model = Model("hist/diapivot.pickle")): """Annotate each lemgram with its corresponding saldo_id according to model. Args: out (str, optional): Resulting annotation file. Defaults to Output("<token>:hist.diapivot", description="SALDO IDs corresponding to lemgrams"). lemgram (str, optional): Existing lemgram annotation. Defaults to Annotation("<token>:saldo.lemgram"). model (str, optional): Crosslink model. Defaults to Model("hist/diapivot.pickle"). """ lexicon = PivotLexicon(model) lemgram_annotation = list(lemgram.read()) out_annotation = [] for lemgrams in lemgram_annotation: saldo_ids = [] for lemgram in lemgrams.split(util.DELIM): s_i = lexicon.get_exactMatch(lemgram) if s_i: saldo_ids += [s_i] out_annotation.append(util.AFFIX + util.DELIM.join(set(saldo_ids)) + util.AFFIX if saldo_ids else util.AFFIX) out.write(out_annotation)
def uppercase( word: Annotation = Annotation("<token:word>"), out: Output = Output("<token>:uppercase.upper"), # some_config_variable: str = Config("uppercase.some_setting") ): """Convert to uppercase.""" out.write([val.upper() for val in word.read()])
def lix(text: Annotation = Annotation("<text>"), sentence: Annotation = Annotation("<sentence>"), word: Annotation = Annotation("<token:word>"), pos: Annotation = Annotation("<token:pos>"), out: Output = Output("<text>:readability.lix", description="LIX values for text chunks"), skip_pos: List[str] = ["MAD", "MID", "PAD"], fmt: str = "%.2f"): """Create LIX annotation for text.""" # Read annotation files and get parent_children relations text_children, _orphans = text.get_children(sentence) word_pos = list(word.read_attributes((word, pos))) sentence_children, _orphans = sentence.get_children(word) sentence_children = list(sentence_children) # Calculate LIX for every text element lix_annotation = [] for text in text_children: in_sentences = [] for sentence_index in text: s = sentence_children[sentence_index] in_sentences.append( list( actual_words([word_pos[token_index] for token_index in s], skip_pos))) lix_annotation.append(fmt % lix_calc(in_sentences)) out.write(lix_annotation)
def translate_tag(out: Output, tag: Annotation, mapping: dict = {}): """Convert part-of-speech tags, specified by the mapping. Example mappings: parole_to_suc, suc_to_simple, ... """ if isinstance(mapping, str): mapping = util.tagsets.mappings[mapping] out.write((mapping.get(t, t) for t in tag.read()))
def find_replace_regex(chunk: Annotation, out: Output, find: str = "", sub: str = ""): """Do find and replace in values of annotation using a regular expressions. N.B: When writing regular expressions in YAML they should be enclosed in single quotes. """ out.write((re.sub(find, sub, val) for val in chunk.read()))
def concat2(out: Output, annotations: List[Annotation] = [Annotation], separator: str = ""): """Concatenate two or more annotations, with an optional separator.""" annotations = [list(a.read()) for a in annotations] out.write([ separator.join([a[n] for a in annotations]) for (n, _) in enumerate(annotations[0]) ])
def msdtag(out: Output = Output( "<token>:hunpos.msd", cls="token:msd", description="Part-of-speeches with morphological descriptions"), word: Annotation = Annotation("<token:word>"), sentence: Annotation = Annotation("<sentence>"), binary: Binary = Binary("[hunpos.binary]"), model: Model = Model("[hunpos.model]"), morphtable: Optional[Model] = Model("[hunpos.morphtable]"), patterns: Optional[Model] = Model("[hunpos.patterns]"), tag_mapping=None, encoding: str = util.UTF8): """POS/MSD tag using the Hunpos tagger.""" if isinstance(tag_mapping, str) and tag_mapping: tag_mapping = util.tagsets.mappings[tag_mapping] elif tag_mapping is None or tag_mapping == "": tag_mapping = {} pattern_list = [] if patterns: with open(patterns.path, encoding="utf-8") as pat: for line in pat: if line.strip() and not line.startswith("#"): name, pattern, tags = line.strip().split("\t", 2) pattern_list.append( (name, re.compile("^%s$" % pattern), tags)) def replace_word(w): """Replace word with alias if word matches a regex pattern.""" for p in pattern_list: if re.match(p[1], w): return "[[%s]]" % p[0] return w sentences, _orphans = sentence.get_children(word) token_word = list(word.read()) stdin = SENT_SEP.join( TOK_SEP.join( replace_word(token_word[token_index]) for token_index in sent) for sent in sentences) args = [model.path] if morphtable: args.extend(["-m", morphtable.path]) stdout, _ = util.system.call_binary(binary, args, stdin, encoding=encoding) out_annotation = word.create_empty_attribute() for sent, tagged_sent in zip(sentences, stdout.strip().split(SENT_SEP)): for token_index, tagged_token in zip( sent, tagged_sent.strip().split(TOK_SEP)): tag = tagged_token.strip().split(TAG_SEP)[TAG_COLUMN] tag = tag_mapping.get(tag, tag) out_annotation[token_index] = tag out.write(out_annotation)
def override(chunk: Annotation, repl: Annotation, out: Output): """Replace values in 'chunk' with non empty values from 'repl'.""" def empty(val): if not val: return True return val == "|" repl = list(repl.read()) out.write((repl[n] if not empty(repl[n]) else val for (n, val) in enumerate(chunk.read())))
def struct_to_token( attr: Annotation = Annotation("{struct}:{attr}"), token: Annotation = Annotation("<token>"), out: Output = Output("<token>:misc.from_struct_{struct}_{attr}")): """Convert an attribute on a structural annotation into a token attribute.""" token_parents = token.get_parents(attr) attr_values = list(attr.read()) out_values = [ attr_values[p] if p is not None else "" for p in token_parents ] out.write(out_values)
def select(out: Output, annotation: Annotation, index: Optional[int] = 0, separator: Optional[str] = " "): """Select a specific index from the values of an annotation. The given annotation values are separated by 'separator', by default whitespace, with at least index + 1 elements. """ if isinstance(index, str): index = int(index) out.write(value.split(separator)[index] for value in annotation.read())
def upostag(out: Output = Output("<token>:misc.upos", cls="token:upos", description="Part-of-speeches in UD"), pos: Annotation = Annotation("<token:pos>")): """Convert SUC POS tags to UPOS.""" pos_tags = pos.read() out_annotation = [] for tag in pos_tags: out_annotation.append(util.tagsets.pos_to_upos(tag, "swe", "SUC")) out.write(out_annotation)
def concat(out: Output, left: Annotation, right: Annotation, separator: str = "", merge_twins: bool = False): """Concatenate values from two annotations, with an optional separator. If merge_twins is set to True, no concatenation will be done on identical values. """ b = list(right.read()) out.write((f"{val_a}{separator}{b[n]}" if not (merge_twins and val_a == b[n]) else val_a for (n, val_a) in enumerate(left.read())))
def annotate( sense: Annotation = Annotation("<token>:saldo.sense"), out_scores: Output = Output("<token>:sensaldo.sentiment_score", description="SenSALDO sentiment score"), out_labels: Output = Output("<token>:sensaldo.sentiment_label", description="SenSALDO sentiment label"), model: Model = Model("[sensaldo.model]"), lexicon=None): """Assign sentiment values to tokens based on their sense annotation. When more than one sense is possible, calulate a weighted mean. - sense: existing annotation with saldoIDs. - out_scores, out_labels: resulting annotation file. - model: pickled lexicon with saldoIDs as keys. - lexicon: this argument cannot be set from the command line, but is used in the catapult. This argument must be last. """ if not lexicon: lexicon = util.PickledLexicon(model.path) # Otherwise use pre-loaded lexicon (from catapult) sense = sense.read() result_scores = [] result_labels = [] for token in sense: # Get set of senses for each token and sort them according to their probabilities token_senses = [ tuple(s.rsplit(util.SCORESEP, 1)) if util.SCORESEP in s else (s, -1.0) for s in token.split(util.DELIM) if s ] token_senses.sort(key=lambda x: float(x[1]), reverse=True) # Lookup the sentiment score for the most probable sense and assign a sentiment label if token_senses: best_sense = token_senses[0][0] score = lexicon.lookup(best_sense, None) else: score = None if score: result_scores.append(score) result_labels.append(SENTIMENT_LABLES.get(int(score))) else: result_scores.append(None) result_labels.append(None) out_scores.write(result_scores) out_labels.write(result_labels)
def contextual(out: Output = Output("{chunk}:geo.geo_context", description="Geographical places with coordinates"), chunk: Annotation = Annotation("{chunk}"), context: Annotation = Annotation("[geo.context_chunk]"), ne_type: Annotation = Annotation("swener.ne:swener.type"), ne_subtype: Annotation = Annotation("swener.ne:swener.subtype"), ne_name: Annotation = Annotation("swener.ne:swener.name"), model: Model = Model("[geo.model]"), method: str = "populous", language: list = []): """Annotate chunks with location data, based on locations contained within the text. context = text chunk to use for disambiguating places (when applicable). chunk = text chunk to which the annotation will be added. """ model = load_model(model, language=language) ne_type_annotation = list(ne_type.read()) ne_subtype_annotation = list(ne_subtype.read()) ne_name_annotation = list(ne_name.read()) children_context_chunk, _orphans = context.get_children(chunk) children_chunk_ne, _orphans = chunk.get_children(ne_type) out_annotation = chunk.create_empty_attribute() for chunks in children_context_chunk: all_locations = [] # TODO: Maybe not needed for anything? context_locations = [] chunk_locations = defaultdict(list) for ch in chunks: for n in children_chunk_ne[ch]: if ne_type_annotation[n] == "LOC" and "PPL" in ne_subtype_annotation[n]: location_text = ne_name_annotation[n].replace("\n", " ").replace(" ", " ") location_data = model.get(location_text.lower()) if location_data: all_locations.append((location_text, list(location_data))) context_locations.append((location_text, list(location_data))) chunk_locations[ch].append((location_text, list(location_data))) else: pass # log.info("No location found for %s" % ne_name_annotation[n].replace("%", "%%")) chunk_locations = most_populous(chunk_locations) for c in chunks: out_annotation[c] = _format_location(chunk_locations.get(c, ())) out.write(out_annotation)
def number_relative(out: Output = Output("{annotation}:misc.number_rel_{parent}"), parent: Annotation = Annotation("{parent}"), child: Annotation = Annotation("{annotation}"), prefix: str = "", zfill: bool = False, start: int = START_DEFAULT): """Number chunks by their relative position within a parent.""" parent_children, _orphans = parent.get_children(child) out.write(("{prefix}{nr:0{length}d}".format(prefix=prefix, length=len(str(len(parent) - 1 + start)) if zfill else 0, nr=cnr) for parent in parent_children for cnr, _index in enumerate(parent, start)))
def ufeatstag(out: Output = Output( "<token>:misc.ufeats", cls="token:ufeats", description="Universal morphological features"), pos: Annotation = Annotation("<token:pos>"), msd: Annotation = Annotation("<token:msd>")): """Convert SUC MSD tags to universal features.""" pos_tags = pos.read() msd_tags = msd.read() out_annotation = [] for pos_tag, msd_tag in zip(pos_tags, msd_tags): feats = util.tagsets.suc_to_feats(pos_tag, msd_tag) out_annotation.append(util.cwbset(feats)) out.write(out_annotation)
def replace_list(chunk: Annotation, out: Output, find: str = "", sub: str = ""): """Find and replace annotations. Find string must match whole annotation. find and sub are whitespace separated lists of words to replace and their replacement. """ find = find.split() sub = sub.split() if len(find) != len(sub): raise util.SparvErrorMessage( "Find and sub must have the same number of words.") translate = dict((f, s) for (f, s) in zip(find, sub)) out.write((translate.get(val, val) for val in chunk.read()))
def process_output(word: Annotation, out: Output, stdout, in_sentences, saldo_annotation, prob_format, default_prob): """Parse WSD output and write annotation.""" out_annotation = word.create_empty_attribute() # Split output into sentences out_sentences = stdout.strip() out_sentences = out_sentences.split("\t".join( ["_", "_", "_", "_", SENT_SEP, "_", "_"])) out_sentences = [i for i in out_sentences if i] # Split output into tokens for out_sent, in_sent in zip(out_sentences, in_sentences): out_tokens = [t for t in out_sent.split("\n") if t] for (out_tok, in_tok) in zip(out_tokens, in_sent): out_prob = out_tok.split("\t")[6] out_prob = [i for i in out_prob.split("|") if i != "_"] out_meanings = [ i for i in out_tok.split("\t")[5].split("|") if i != "_" ] saldo = [ i for i in saldo_annotation[in_tok].strip(util.AFFIX).split( util.DELIM) if i ] new_saldo = [] if out_prob: for meaning in saldo: if meaning in out_meanings: i = out_meanings.index(meaning) new_saldo.append((meaning, float(out_prob[i]))) else: new_saldo.append((meaning, default_prob)) else: new_saldo = [(meaning, default_prob) for meaning in saldo] # Sort by probability new_saldo.sort(key=lambda x: (-x[1], x[0])) # Format probability according to prob_format new_saldo = [ saldo + prob_format % prob if prob_format else saldo for saldo, prob in new_saldo ] out_annotation[in_tok] = util.cwbset(new_saldo) out.write(out_annotation)
def ids(doc: Document = Document(), annotation: Annotation = Annotation("{annotation}"), out: Output = Output("{annotation}:misc.id", description="Unique ID for {annotation}"), docid: AnnotationData = AnnotationData("<docid>"), prefix: str = ""): """Create unique IDs for every span of an existing annotation.""" docid = docid.read() prefix = prefix + docid ann = list(annotation.read()) out_annotation = [] # Use doc name and annotation name as seed for the IDs _reset_id("{}/{}".format(doc, annotation), len(ann)) for _ in ann: new_id = _make_id(prefix, out_annotation) out_annotation.append(new_id) out.write(out_annotation)
def nominal_ratio(text: Annotation = Annotation("<text>"), pos: Annotation = Annotation("<token:pos>"), out: Output = Output( "<text>:readability.nk", description="Nominal ratios for text chunks"), noun_pos: List[str] = ["NN", "PP", "PC"], verb_pos: List[str] = ["PN", "AB", "VB"], fmt: str = "%.2f"): """Create nominal ratio annotation for text.""" text_children, _orphans = text.get_children(pos) pos_annotation = list(pos.read()) # Calculate OVIX for every text element nk_annotation = [] for text in text_children: in_pos = [pos_annotation[token_index] for token_index in text] nk_annotation.append(fmt % nominal_ratio_calc(in_pos, noun_pos, verb_pos)) out.write(nk_annotation)
def ovix(text: Annotation = Annotation("<text>"), word: Annotation = Annotation("<token:word>"), pos: Annotation = Annotation("<token:pos>"), out: Output = Output("<text>:readability.ovix", description="OVIX values for text chunks"), skip_pos: List[str] = ["MAD", "MID", "PAD"], fmt: str = "%.2f"): """Create OVIX annotation for text.""" text_children, _orphans = text.get_children(word) word_pos = list(word.read_attributes((word, pos))) # Calculate OVIX for every text element ovix_annotation = [] for text in text_children: in_words = list( actual_words([word_pos[token_index] for token_index in text], skip_pos)) ovix_annotation.append(fmt % ovix_calc(in_words)) out.write(ovix_annotation)
def metadata(out: Output = Output("{chunk}:geo.geo_metadata", description="Geographical places with coordinates"), chunk: Annotation = Annotation("{chunk}"), source: Annotation = Annotation("[geo.metadata_source]"), model: Model = Model("[geo.model]"), method: str = "populous", language: list = []): """Get location data based on metadata containing location names.""" geomodel = load_model(model, language=language) same_target_source = chunk.split()[0] == source.split()[0] chunk_annotation = list(chunk.read()) source_annotation = list(source.read()) # If location source and target chunk are not the same, we need # to find the parent/child relations between them. if not same_target_source: target_source_parents = list(source.get_parents(chunk)) chunk_locations = {} for i, _ in enumerate(chunk_annotation): if same_target_source: location_source = source_annotation[i] else: location_source = source_annotation[target_source_parents[i]] if target_source_parents[ i] is not None else None if location_source: location_data = geomodel.get(location_source.strip().lower()) if location_data: chunk_locations[i] = [(location_source, list(location_data))] else: chunk_locations[i] = [] chunk_locations = most_populous(chunk_locations) out_annotation = chunk.create_empty_attribute() for c in chunk_locations: out_annotation[c] = _format_location(chunk_locations.get(c, ())) out.write(out_annotation)
def _read_chunks_and_write_new_ordering(out: Output, chunk: Annotation, order, prefix="", zfill=False, start=START_DEFAULT): """Common function called by other numbering functions.""" new_order = defaultdict(list) in_annotation = list(chunk.read()) for i, val in enumerate(in_annotation): val = order(i, val) new_order[val].append(i) out_annotation = chunk.create_empty_attribute() nr_digits = len(str(len(new_order) - 1 + start)) for nr, key in enumerate(sorted(new_order), start): for index in new_order[key]: out_annotation[index] = "{prefix}{nr:0{length}d}".format(prefix=prefix, length=nr_digits if zfill else 0, nr=nr) out.write(out_annotation)
def text_spans(text: Text = Text(), chunk: Annotation = Annotation("<token>"), out: Output = Output("<token>:misc.word", cls="token:word"), keep_formatting_chars: Optional[bool] = Config( "misc.keep_formatting_chars")): """Add the text content for each edge as a new annotation.""" corpus_text = text.read() if isinstance(chunk, (str, Annotation)): chunk = chunk.read_spans() out_annotation = [] for span in chunk: token = corpus_text[span[0]:span[1]] if not keep_formatting_chars: new_token = util.remove_formatting_characters(token) # If this token consists entirely of formatting characters, don't remove them. Empty tokens are bad! if new_token: token = new_token out_annotation.append(token) if out: out.write(out_annotation) else: return out_annotation
def text_headtail(text: Text = Text(), chunk: Annotation = Annotation("<token>"), out_head: Output = Output("<token>:misc.head"), out_tail: Output = Output("<token>:misc.tail")): """Extract "head" and "tail" whitespace characters for tokens.""" def escape(t): """Escape whitespace characters.""" return t.replace(" ", "\\s").replace("\n", "\\n").replace("\t", "\\t") out_head_annotation = chunk.create_empty_attribute() out_tail_annotation = chunk.create_empty_attribute() head_text = None corpus_text = text.read() chunk = list(chunk.read()) for i, span in enumerate(chunk): if head_text: out_head_annotation[i] = escape(head_text) head_text = None if i < len(chunk) - 1: tail_start = span[1][0] tail_end = chunk[i + 1][0][0] tail_text = corpus_text[tail_start:tail_end] try: n_pos = tail_text.rindex("\n") except ValueError: n_pos = None if n_pos is not None and n_pos + 1 < len(tail_text): head_text = tail_text[n_pos + 1:] tail_text = tail_text[:n_pos + 1] if tail_text: out_tail_annotation[i] = escape(tail_text) out_head.write(out_head_annotation) out_tail.write(out_tail_annotation)
def annotate( lang: Language = Language(), model: Model = Model("[treetagger.model]"), tt_binary: Binary = Binary("[treetagger.binary]"), out_upos: Output = Output("<token>:treetagger.upos", cls="token:upos", description="Part-of-speeches in UD"), out_pos: Output = Output( "<token>:treetagger.pos", cls="token:pos", description="Part-of-speeches from TreeTagger"), out_baseform: Output = Output("<token>:treetagger.baseform", description="Baseforms from TreeTagger"), word: Annotation = Annotation("<token:word>"), sentence: Annotation = Annotation("<sentence>"), encoding: str = util.UTF8): """POS/MSD tag and lemmatize using TreeTagger.""" sentences, _orphans = sentence.get_children(word) word_annotation = list(word.read()) stdin = SENT_SEP.join( TOK_SEP.join(word_annotation[token_index] for token_index in sent) for sent in sentences) args = ["-token", "-lemma", "-no-unknown", "-eos-tag", "<eos>", model.path] stdout, stderr = util.system.call_binary(tt_binary, args, stdin, encoding=encoding) log.debug("Message from TreeTagger:\n%s", stderr) # Write pos and upos annotations. out_upos_annotation = word.create_empty_attribute() out_pos_annotation = word.create_empty_attribute() for sent, tagged_sent in zip(sentences, stdout.strip().split(SENT_SEP)): for token_id, tagged_token in zip(sent, tagged_sent.strip().split(TOK_SEP)): tag = tagged_token.strip().split(TAG_SEP)[TAG_COLUMN] out_pos_annotation[token_id] = tag out_upos_annotation[token_id] = util.tagsets.pos_to_upos( tag, lang, TAG_SETS.get(lang)) out_pos.write(out_pos_annotation) out_upos.write(out_upos_annotation) # Write lemma annotations. out_lemma_annotation = word.create_empty_attribute() for sent, tagged_sent in zip(sentences, stdout.strip().split(SENT_SEP)): for token_id, tagged_token in zip(sent, tagged_sent.strip().split(TOK_SEP)): lem = tagged_token.strip().split(TAG_SEP)[LEM_COLUMN] out_lemma_annotation[token_id] = lem out_baseform.write(out_lemma_annotation)
def annotate( maltjar: Binary = Binary("[malt.jar]"), model: Model = Model("[malt.model]"), out_dephead: Output = Output( "<token>:malt.dephead", cls="token:dephead", description="Positions of the dependency heads"), out_dephead_ref: Output = Output( "<token>:malt.dephead_ref", cls="token:dephead_ref", description="Sentence-relative positions of the dependency heads"), out_deprel: Output = Output( "<token>:malt.deprel", cls="token:deprel", description="Dependency relations to the head"), word: Annotation = Annotation("<token:word>"), pos: Annotation = Annotation("<token:pos>"), msd: Annotation = Annotation("<token:msd>"), ref: Annotation = Annotation("<token>:misc.number_rel_<sentence>"), sentence: Annotation = Annotation("<sentence>"), token: Annotation = Annotation("<token>"), encoding: str = util.UTF8, process_dict=None): """ Run the malt parser, in an already started process defined in process_dict, or start a new process (default). The process_dict argument should never be set from the command line. """ if process_dict is None: process = maltstart(maltjar, model, encoding) else: process = process_dict["process"] # If process seems dead, spawn a new if process.stdin.closed or process.stdout.closed or process.poll(): util.system.kill_process(process) process = maltstart(maltjar, model, encoding, send_empty_sentence=True) process_dict["process"] = process sentences, orphans = sentence.get_children(token) sentences.append(orphans) word_annotation = list(word.read()) pos_annotation = list(pos.read()) msd_annotation = list(msd.read()) ref_annotation = list(ref.read()) def conll_token(nr, token_index): form = word_annotation[token_index] lemma = UNDEF pos = cpos = pos_annotation[token_index] feats = re.sub(r"[ ,.]", "|", msd_annotation[token_index]).replace("+", "/") return TAG_SEP.join((str(nr), form, lemma, cpos, pos, feats)) stdin = SENT_SEP.join( TOK_SEP.join( conll_token(n + 1, token_index) for n, token_index in enumerate(sent)) for sent in sentences) if encoding: stdin = stdin.encode(encoding) keep_process = len( stdin) < RESTART_THRESHOLD_LENGTH and process_dict is not None log.info("Stdin length: %s, keep process: %s", len(stdin), keep_process) if process_dict is not None: process_dict["restart"] = not keep_process if keep_process: # Chatting with malt: send a SENT_SEP and read correct number of lines stdin_fd, stdout_fd = process.stdin, process.stdout stdin_fd.write(stdin + SENT_SEP.encode(util.UTF8)) stdin_fd.flush() malt_sentences = [] for sent in sentences: malt_sent = [] for _ in sent: line = stdout_fd.readline() if encoding: line = line.decode(encoding) malt_sent.append(line) line = stdout_fd.readline() assert line == b"\n" malt_sentences.append(malt_sent) else: # Otherwise use communicate which buffers properly stdout, _ = process.communicate(stdin) if encoding: stdout = stdout.decode(encoding) malt_sentences = (malt_sent.split(TOK_SEP) for malt_sent in stdout.split(SENT_SEP)) out_dephead_annotation = word.create_empty_attribute() out_dephead_ref_annotation = out_dephead_annotation.copy() out_deprel_annotation = out_dephead_annotation.copy() for (sent, malt_sent) in zip(sentences, malt_sentences): for (token_index, malt_tok) in zip(sent, malt_sent): cols = [(None if col == UNDEF else col) for col in malt_tok.split(TAG_SEP)] out_deprel_annotation[token_index] = cols[DEPREL_COLUMN] head = int(cols[HEAD_COLUMN]) out_dephead_annotation[token_index] = str(sent[head - 1]) if head else "-" out_dephead_ref_annotation[token_index] = str( ref_annotation[sent[head - 1]]) if head else "" out_dephead.write(out_dephead_annotation) out_dephead_ref.write(out_dephead_ref_annotation) out_deprel.write(out_deprel_annotation)
def annotate_words(out: Output, model: Model, saldoids: Annotation, pos: Annotation, annotate, pos_limit: List[str], class_set=None, disambiguate=True, connect_ids=False, delimiter=util.DELIM, affix=util.AFFIX, scoresep=util.SCORESEP, lexicon=None): """ Annotate words with blingbring classes (rogetID). - out_sent: resulting annotation file. - model: pickled lexicon with saldoIDs as keys. - saldoids, pos: existing annotation with saldoIDs/parts of speech. - annotate: annotation function, returns an iterable containing annotations for one token ID. (annotate_bring() or annotate_swefn()) - pos_limit: parts of speech that will be annotated. Set to None to annotate all pos. - class_set: output Bring classes or Roget IDs ("bring", "roget_head", "roget_subsection", "roget_section" or "roget_class"). Set to None when not annotating blingbring. - disambiguate: use WSD and use only the most likely saldo ID. - connect_IDs: for sweFN: paste saldo ID after each sweFN ID. - delimiter: delimiter character to put between ambiguous results - affix: optional character to put before and after results to mark a set. - lexicon: this argument cannot be set from the command line, but is used in the catapult. This argument must be last. """ if not lexicon: lexicon = util.PickledLexicon(model.path) # Otherwise use pre-loaded lexicon (from catapult) sense = saldoids.read() token_pos = list(pos.read()) out_annotation = pos.create_empty_attribute() # Check if the saldo IDs are ranked (= word senses have been disambiguated) wsd = saldoids.split()[1].split(".")[0] == "wsd" for token_index, token_sense in enumerate(sense): # Check if part of speech of this token is allowed if not pos_ok(token_pos, token_index, pos_limit): saldo_ids = None out_annotation[token_index] = affix continue if wsd and util.SCORESEP in token_sense: ranked_saldo = token_sense.strip(util.AFFIX).split(util.DELIM) \ if token_sense != util.AFFIX else None saldo_tuples = [(i.split(util.SCORESEP)[0], i.split(util.SCORESEP)[1]) for i in ranked_saldo] if not disambiguate: saldo_ids = [i[0] for i in saldo_tuples] # Only take the most likely analysis into account. # Handle wsd with equal probability for several words else: saldo_ids = [saldo_tuples[0]] del saldo_tuples[0] while saldo_tuples and (saldo_tuples[0][1] == saldo_ids[0][1]): saldo_ids = [saldo_tuples[0]] del saldo_tuples[0] saldo_ids = [i[0] for i in saldo_ids] else: # No WSD saldo_ids = token_sense.strip(util.AFFIX).split(util.DELIM) \ if token_sense != util.AFFIX else None result = annotate(saldo_ids, lexicon, connect_ids, scoresep) out_annotation[token_index] = util.cwbset(result, delimiter, affix) if result else affix out.write(out_annotation)
def do_segmentation(text: Text, out: Output, segmenter, chunk: Optional[Annotation] = None, existing_segments=None, model: Optional[Model] = None, token_list: Optional[Model] = None): """Segment all chunks (e.g. sentences) into smaller "tokens" (e.g. words), and annotate them as "element" (e.g. w). Segmentation is done by the given "segmenter"; some segmenters take an extra argument which is a pickled "model" object. """ segmenter_args = [] if model: if model.path.suffix in ["pickle", "pkl"]: with open(model, "rb") as M: model_arg = pickle.load(M, encoding="UTF-8") else: model_arg = str(model.path) segmenter_args.append(model_arg) assert segmenter in SEGMENTERS, "Available segmenters: %s" % ", ".join( sorted(SEGMENTERS)) segmenter = SEGMENTERS[segmenter] segmenter = segmenter(*segmenter_args) assert hasattr( segmenter, "span_tokenize" ), "Segmenter needs a 'span_tokenize' method: %r" % segmenter corpus_text = text.read() # First we read the chunks and partition the text into spans # E.g., "one two <s>three four</s> five <s>six</s>" # ==> ["one two ", "three four", " five ", "six"] # (but using spans (pairs of anchors) instead of strings) positions = set() chunk_spans = chunk.read_spans() if chunk else [] positions = positions.union( set(pos for span in chunk_spans for pos in span)) positions = sorted({0, len(corpus_text)} | positions) chunk_spans = list(zip(positions, positions[1:])) if existing_segments: segments = list(existing_segments.read_spans()) for n, (chunk_start, chunk_end) in enumerate(chunk_spans[:]): for segment_start, segment_end in segments: if segment_end <= chunk_start: continue if segment_start >= chunk_end: break if chunk_start != segment_start: chunk_spans.append((chunk_start, segment_start)) chunk_start = segment_end chunk_spans[n] = (chunk_start, chunk_end) chunk_spans.sort() log.info("Reorganized into %d chunks" % len(chunk_spans)) else: segments = [] # Now we can segment each chunk span into tokens for start, end in chunk_spans: for spanstart, spanend in segmenter.span_tokenize( corpus_text[start:end]): spanstart += start spanend += start if corpus_text[spanstart:spanend].strip(): span = (spanstart, spanend) segments.append(span) segments.sort() out.write(segments)
def _formatter(in_from: Annotation, in_to: Optional[Annotation], out_from: Output, out_to: Output, informat: str, outformat: str, splitter: str, regex: str): """Take existing dates/times and input formats and convert to specified output format.""" def get_smallest_unit(informat): smallest_unit = 0 # No date if "%y" not in informat and "%Y" not in informat: pass elif "%b" not in informat and "%B" not in informat and "%m" not in informat: smallest_unit = 1 # year elif "%d" not in informat: smallest_unit = 2 # month elif "%H" not in informat and "%I" not in informat: smallest_unit = 3 # day elif "%M" not in informat: smallest_unit = 4 # hour elif "%S" not in informat: smallest_unit = 5 # minute else: smallest_unit = 6 # second return smallest_unit def get_date_length(informat): parts = informat.split("%") length = len( parts[0]) # First value is either blank or not part of date lengths = { "Y": 4, "3Y": 3, "y": 2, "m": 2, "b": None, "B": None, "d": 2, "H": None, "I": None, "M": 2, "S": 2 } for part in parts[1:]: add = lengths.get(part[0], None) if add: length += add + len(part[1:]) else: return None return length if not in_to: in_to = in_from informat = informat.split("|") outformat = outformat.split("|") if splitter: splitter = splitter assert len(outformat) == 1 or (len(outformat) == len(informat)), "The number of out-formats must be equal to one " \ "or the number of in-formats." ifrom = list(in_from.read()) ofrom = in_from.create_empty_attribute() for index, val in enumerate(ifrom): val = val.strip() if not val: ofrom[index] = None continue tries = 0 for inf in informat: if splitter and splitter in inf: values = re.findall("%[YybBmdHMS]", inf) if len(set(values)) < len(values): vals = val.split(splitter) inf = inf.split(splitter) else: vals = [val] inf = [inf] if regex: temp = [] for v in vals: matches = re.search(regex, v) if matches: temp.append([x for x in matches.groups() if x][0]) if not temp: # If the regex doesn't match, treat as no date ofrom[index] = None continue vals = temp tries += 1 try: fromdates = [] for i, v in enumerate(vals): if "%3Y" in inf[i]: datelen = get_date_length(inf[i]) if datelen and not datelen == len(v): raise ValueError inf[i] = inf[i].replace("%3Y", "%Y") v = "0" + v if "%0m" in inf[i] or "%0d" in inf[i]: inf[i] = inf[i].replace("%0m", "%m").replace("%0d", "%d") datelen = get_date_length(inf[i]) if datelen and not datelen == len(v): raise ValueError fromdates.append(datetime.datetime.strptime(v, inf[i])) if len(fromdates) == 1 or out_to: ofrom[index] = fromdates[0].strftime(outformat[0] if len( outformat) == 1 else outformat[tries - 1]) else: outstrings = [ fromdate.strftime(outformat[0] if len(outformat) == 1 else outformat[tries - 1]) for fromdate in fromdates ] ofrom[index] = outstrings[0] + splitter + outstrings[1] break except ValueError: if tries == len(informat): log.error("Could not parse: %s", str(vals)) raise continue out_from.write(ofrom) del ofrom if out_to: ito = list(in_to.read()) oto = in_to.create_empty_attribute() for index, val in enumerate(ito): if not val: oto[index] = None continue tries = 0 for inf in informat: if splitter and splitter in inf: values = re.findall("%[YybBmdHMS]", inf) if len(set(values)) < len(values): vals = val.split(splitter) inf = inf.split(splitter) else: vals = [val] inf = [inf] if regex: temp = [] for v in vals: matches = re.search(regex, v) if matches: temp.append([x for x in matches.groups() if x][0]) if not temp: # If the regex doesn't match, treat as no date oto[index] = None continue vals = temp tries += 1 try: todates = [] for i, v in enumerate(vals): if "%3Y" in inf[i]: datelen = get_date_length(inf[i]) if datelen and not datelen == len(v): raise ValueError inf[i] = inf[i].replace("%3Y", "%Y") v = "0" + v if "%0m" in inf[i] or "%0d" in inf[i]: inf[i] = inf[i].replace("%0m", "%m").replace("%0d", "%d") datelen = get_date_length(inf[i]) if datelen and not datelen == len(v): raise ValueError todates.append(datetime.datetime.strptime(v, inf[i])) smallest_unit = get_smallest_unit(inf[0]) if smallest_unit == 1: add = relativedelta(years=1) elif smallest_unit == 2: add = relativedelta(months=1) elif smallest_unit == 3: add = relativedelta(days=1) elif smallest_unit == 4: add = relativedelta(hours=1) elif smallest_unit == 5: add = relativedelta(minutes=1) elif smallest_unit == 6: add = relativedelta(seconds=1) todates = [ todate + add - relativedelta(seconds=1) for todate in todates ] oto[index] = todates[-1].strftime(outformat[0] if len( outformat) == 1 else outformat[tries - 1]) break except ValueError: if tries == len(informat): log.error("Could not parse: %s", str(vals)) raise continue out_to.write(oto)