def main(args): if not args.as_array and not args.as_extra: args.as_extra = True for spec in read_specs(args, converters=FROM_FORMAT_NO_PLACEHOLDERS): kwargs = dict(as_array=args.as_array, as_extra=args.as_extra, verbose=args.verbose, lang=spec.lang) passages = spec.passages if spec.conllu: passages = copy_annotation(passages, spec.conllu, by_id=args.by_id, **kwargs) elif spec.udpipe: passages = annotate_udpipe(passages, spec.udpipe, **kwargs) elif spec.stanfordnlp: passages = annotate_stanfordnlp(passages, spec.stanfordnlp, **kwargs) for passage in annotate_all(passages if args.verbose else tqdm( passages, unit=" passages", desc="Annotating " + spec.out_dir), replace=spec.conllu or not (spec.udpipe or spec.stanfordnlp), **kwargs): if passage.extra.get("format") == "amr" and args.as_array: from semstr.conversion.amr import AmrConverter AmrConverter.introduce_placeholders(passage) write_passage(passage, outdir=spec.out_dir, verbose=args.verbose, binary=args.binary)
def main(args): words = args.word or [] categories = list(args.category or ()) dependencies = list(args.dependency or ()) if args.case_insensitive: words = list(map(str.lower, words)) for spec in read_specs(args, converters=FROM_FORMAT): if args.dependency: spec.passages = annotate_udpipe(spec.passages, spec.udpipe) if spec.udpipe else \ annotate_all(spec.passages, as_array=True, replace=not spec.udpipe, lang=spec.lang) t = tqdm(spec.passages, unit=" passages", desc="Finding") if words: t.set_postfix(words=",".join(words)) if categories: t.set_postfix(categories=",".join(categories)) if dependencies: t.set_postfix(dependencies=",".join(dependencies)) found = 0 filename = os.path.join( spec.out_dir, "_".join(words + categories + dependencies) + ".txt") with open(filename, "w", encoding="utf-8") as f: for passage in t: for terminal in passage.layer(layer0.LAYER_ID).all: parent = terminal.parents[0] word = terminal.text if args.case_insensitive: word = word.lower() if (not words or word in words) and ( not categories or parent.ftag in categories) and ( not dependencies or get_annotation( terminal, spec.udpipe) in dependencies): print(passage.ID, parent.fparent, file=f) found += 1 t.set_postfix(found=found) print("Wrote '%s'" % filename)
def main(args): for spec in read_specs(args, converters=CONVERTERS): scores = [] sentences, to_parse = tee( (to_conllu_native(p), to_conllu_native(p, test=True, enhanced=False) ) if isinstance(p, core.Passage) else (p, strip_enhanced(p)) for p in spec.passages) t = tqdm(zip((x for x, _ in sentences), split_by_empty_lines( udpipe((x for _, x in to_parse), spec.udpipe, args.verbose))), unit=" sentences") for sentence, parsed in t: sentence = list(sentence) if args.write: i = next(find_ids(sentence)) t.set_postfix(id=i) with open_out_file(spec, i) as f: for line in parsed: print(line, file=f) if args.evaluate: scores.append( evaluate(parsed, sentence, verbose=args.verbose > 1)) if scores: Scores(scores).print()
def main(args): for spec in read_specs(args, converters=FROM_FORMAT): scores = [] if not args.verbose: spec.passages = tqdm( spec.passages, unit=" passages", desc="Parsing " + (spec.out_dir if spec.out_dir != "." else spec.lang)) for passage, parsed in parse(spec.passages, spec.lang, spec.udpipe, args.verbose): map_labels(parsed, args.label_map) normalize(parsed, extra=True) if args.write: write_passage(parsed, args) if args.evaluate: evaluator = EVALUATORS.get(args.output_format) converter = TO_FORMAT.get(args.output_format) if converter is not None: passage, parsed = map(converter, (passage, parsed)) if evaluator is not None: scores.append( evaluator.evaluate(parsed, passage, verbose=args.verbose > 1)) if scores: Scores(scores).print()
def main(args): for spec in read_specs(args, converters=FROM_FORMAT): spec.passages = annotate_udpipe(spec.passages, spec.udpipe) if spec.udpipe else \ annotate_all(spec.passages, as_array=True, replace=not spec.udpipe, lang=spec.lang) filename = os.path.join(spec.out_dir, "find.db") with sqlite3.connect(filename) as conn: c = conn.cursor() c.execute("DROP TABLE terminals") c.execute( "CREATE TABLE terminals (pid, tid, text, ftag, fparent, dep)") c.execute("CREATE INDEX idx_terminals_pid ON terminals (pid)") c.execute("CREATE INDEX idx_terminals_text ON terminals (text)") c.execute("CREATE INDEX idx_terminals_ftag ON terminals (ftag)") c.execute("CREATE INDEX idx_terminals_dep ON terminals (dep)") for passage in tqdm(spec.passages, unit=" passages", desc="Creating " + filename): rows = [] for terminal in passage.layer(layer0.LAYER_ID).all: parent = terminal.parents[0] rows.append( (passage.ID, terminal.ID, terminal.text, parent.ftag, str(parent.fparent), get_annotation(terminal, spec.udpipe))) c.executemany("INSERT INTO terminals VALUES (?,?,?,?,?,?)", rows) conn.commit()
def main(args): for spec in read_specs(args, converters=FROM_FORMAT): if spec.udpipe: spec.passages = annotate_udpipe(spec.passages, spec.udpipe, as_array=args.as_array, verbose=args.verbose) elif spec.conllu: spec.passages = copy_annotation(spec.passages, spec.conllu, as_array=args.as_array, verbose=args.verbose) for passage in annotate_all(spec.passages if args.verbose else tqdm(spec.passages, unit=" passages", desc="Annotating " + spec.out_dir), as_array=args.as_array, replace=not spec.udpipe, lang=spec.lang, verbose=args.verbose): write_passage(passage, outdir=spec.out_dir, verbose=args.verbose, binary=args.binary)
def main(args): categories = "".join(args.category) for spec in read_specs(args, converters=FROM_FORMAT): t = tqdm(spec.passages, unit=" passages", desc="Finding multi-token units", postfix={"categories": categories}) found = 0 filename = os.path.join(spec.out_dir, "multi_token_" + categories + ".txt") with open(filename, "w", encoding="utf-8") as f: for passage in t: for node in passage.layer(layer1.LAYER_ID).all: try: if node.ftag in categories and len( node.get_terminals(remotes=True)) > 1: print(passage.ID, node.ID, node, file=f) found += 1 t.set_postfix(found=found) except (AttributeError, KeyError, ValueError, TypeError): pass print("Wrote '%s'" % filename)
def main(args): for spec in read_specs(args, converters=CONVERTERS): scores = [] sentences1, sentences2 = tee(spec.passages) t = tqdm(zip( sentences1, split_by_empty_lines(udpipe(sentences2, spec.udpipe, args.verbose))), unit=" sentences") for sentence, parsed in t: sentence = list(sentence) if args.write: i = next(find_ids(sentence)) t.set_postfix(id=i) with open(os.path.join(spec.out_dir, i + ".conllu"), "w", encoding="utf-8") as f: for line in parsed: print(line, file=f) if args.evaluate: scores.append( evaluate(parsed, sentence, verbose=args.verbose > 1)) if scores: Scores(scores).print()
def main(args): for spec in read_specs(args, converters=FROM_FORMAT): for passage in tqdm(spec.passages, unit=" passages", desc="Setting language in " + spec.out_dir, postfix={"lang": spec.lang}): passage.attrib["lang"] = spec.lang write_passage(passage, outdir=spec.out_dir, verbose=False, binary=args.binary)