#!/usr/bin/python

# -*- coding: utf-8 -*-

import re, sys, io, os, platform

import tempfile

import subprocess

from glob import glob

from lib._version import __version__

from lib.stacked_tokenizer import StackedTokenizer

from lib.order_meta_tag import reorder

from lib.ekthetic_para import ekthetic_to_para

from lib.tt2conll import conllize

from lib.depedit import DepEdit

from lib.binarize_tags import binarize

from lib.lang import lookup_lang

from lib.harvest_tt_sgml import harvest_tt

from lib.mwe import tag_mwes

PY3 = sys.version_info[0] > 2

if PY3:

from builtins import type

inp = input if PY3 else raw_input

script_dir = os.path.dirname(os.path.realpath(__file__))

lib_dir = script_dir + os.sep + "lib" + os.sep

bin_dir = script_dir + os.sep + "bin" + os.sep

data_dir = script_dir + os.sep + "data" + os.sep

parser_path = bin_dir + "maltparser-1.8" + os.sep

tt_path = bin_dir + "TreeTagger" + os.sep + "bin" + os.sep

def log_tasks(opts):

sys.stderr.write("\n")

def groupify(output,anno):

return groups

def remove_nesting_attr(data, nester, nested, attr="xml:lang"):

return "\n".join(lines)

def tok_from_norm(data):

return "\n".join(outdata) + "\n"

def read_attributes(input,attribute_name):

return out_stream

def merge_into_tag(tag_to_kill, tag_to_merge_into,stream):

return injected

def exec_via_temp(input_text, command_params, workdir=""):

return exec_out

def get_origs(data):

return "\n".join(origs)

def inject(attribute_name, contents, at_attribute,into_stream,replace=True):

return injected

def extract_conll(conll_string):

return ids, funcs, parents

def space_punct(input_text):

return outstr

def inject_tags(in_sgml,insertion_specs,around_tag="norm",inserted_tag="multiword"):

return "\n".join(outlines)

def check_requirements():

return tt_OK, malt_OK

def download_requirements(tt_ok=True, malt_ok=True):

shutil.copyfile(bin_dir+"coptic_fine.par",bin_dir+"TreeTagger" + os.sep + "bin" + os.sep + "coptic_fine.par")

def nlp_coptic(input_data, lb=False, parse_only=False, do_tok=True, do_norm=True, do_mwe=True, do_tag=True, do_lemma=True, do_lang=True,

return output.strip() + "\n"

if __name__ == "__main__":

if sys.version_info[0] == 2 and sys.version_info[1] < 7:

sys.stderr.write("Python versions below 2.7 are not supported.\n")

sys.stderr.write("Your Python version:\n")

sys.stderr.write(".".join([str(v) for v in sys.version_info[:3]]) + "\n")

sys.exit(0)

from argparse import ArgumentParser, RawDescriptionHelpFormatter

parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter)

parser.prog = "Coptic NLP Pipeline"

parser.usage = "python coptic_nlp.py [OPTIONS] files"

parser.epilog = """Example usage:

--------------

Add norm, lemma, parse, tag, unary tags, find multiword expressions and do language recognition:

> python coptic_nlp.py -penmult infile.txt

Just tokenize a file using pipes and dashes:

> python coptic_nlp.py -o pipes infile.txt

Tokenize with pipes and mark up line breaks, conservatively detokenize bound groups, assume seg boundary at merge site:

> python coptic_nlp.py -b -d 1 --segment_merged -o pipes infile.txt

Normalize, tag, lemmatize, find multiword expressions and parse, splitting sentences by <verse> tags:

> python coptic_nlp.py -pnltm -s verse infile.txt

Add full analyses to a whole directory of *.xml files, output to a specified directory:

> python coptic_nlp.py -penmult --dirout /home/cop/out/ *.xml

Parse a tagged SGML file into CoNLL tabular format for treebanking, use translation tag to recognize sentences:

> python coptic_nlp.py --no_tok --parse_only --pos_spans -s translation infile.tt

Merge a parse into a tagged SGML file's <norm> tags, use translation tag to recognize sentences:

> python coptic_nlp.py --merge_parse --pos_spans -s translation infile.tt

"""

parser.add_argument("files", help="File name or pattern of files to process (e.g. *.txt)")

g1 = parser.add_argument_group("standard module options")

g1.add_argument("-u","--unary", action="store_true", help='Binarize unary XML milestone tags')

g1.add_argument("-t","--tag", action="store_true", help='Do POS tagging')

g1.add_argument("-l","--lemma", action="store_true", help='Do lemmatization')

g1.add_argument("-n","--norm", action="store_true", help='Do normalization')

g1.add_argument("-m","--multiword", action="store_true", help='Tag multiword expressions')

g1.add_argument("-b","--breaklines", action="store_true", help='Add line tags at line breaks')

g1.add_argument("-p","--parse", action="store_true", help='Parse with dependency parser')

g1.add_argument("-e","--etym", action="store_true", help='Add etymolgical language of origin for loan words')

g1.add_argument("-s","--sent", action="store", help='XML tag to split sentences, e.g. verse for <verse ..> (otherwise PUNCT tag is used to split sentences)')

g1.add_argument("-o","--outmode", action="store", choices=["pipes","sgml","conllu"], default="sgml", help='Output SGML, conllu or tokenize with pipes')

g2 = parser.add_argument_group("less common options")

g2.add_argument("-f","--finitestate", action="store_true", help='Use old finite-state tokenizer (less accurate)')

g2.add_argument("-d","--detokenize", action="store", type=int, choices=[0,1,2], default=0, help="Re-group non-standard bound groups (a.k.a. 'laytonize') - 1=normal 2=aggressive")

g2.add_argument("--segment_merged", action="store_true", help="When re-grouping bound groups, assume merged groups have segmentation boundary between them")

g2.add_argument("-q","--quiet", action="store_true", help='Suppress verbose messages')

g2.add_argument("-x","--extension", action="store", default=None, help='Extension for SGML mode output files (default: tt)')

g2.add_argument("--stdout", action="store_true", help='Print output to stdout, do not create output file')

g2.add_argument("--para", action="store_true", help='Add <p> tags if <hi rend="ekthetic"> is present')

g2.add_argument("--space", action="store_true", help='Add spaces around punctuation')

g2.add_argument("--from_pipes", action="store_true", help='Tokenization is indicated in input via pipes')

g2.add_argument("--dirout", action="store", default=".", help='Optional output directory (default: this dir)')

g2.add_argument("--meta", action="store", default=None, help='Add fixed meta data string read from this file name')

g2.add_argument("--parse_only", action="store_true", help='Only add a parse to an existing tagged SGML input')

g2.add_argument("--no_tok", action="store_true", help='Do not tokenize at all, input is one token per line')

g2.add_argument("--pos_spans", action="store_true", help='Harvest POS tags and lemmas from SGML spans')

g2.add_argument("--merge_parse", action="store_true", help='Merge/add a parse into a ready SGML file')

g2.add_argument("--version", action="store_true", help='Print version number and quit')

if "--version" in sys.argv:

sys.stdout.write("Coptic NLP Pipeline V" + __version__)

sys.exit(1)

opts = parser.parse_args()

old_tokenizer = True if opts.finitestate else False

dotok = False if opts.no_tok or opts.merge_parse else True

if not opts.quiet:

from lib import timing

add_fixed_meta = ""

if opts.meta is not None:

add_fixed_meta = io.open(opts.meta, encoding="utf8").read()

files = glob(opts.files)

if not opts.quiet:

log_tasks(opts)

if dotok and not old_tokenizer:

# Pre-load stacked tokenizer for entire batch

stk = StackedTokenizer(pipes=opts.outmode == "pipes", lines=opts.breaklines, tokenized=opts.from_pipes,

segment_merged=opts.segment_merged, detok=opts.detokenize)

else:

stk = None

# Check if TreeTagger and Malt Parser are available

tt_OK, malt_OK = check_requirements()

if (opts.tag and not tt_OK) or ((opts.parse or opts.parse_only or opts.merge_parse) and not malt_OK):

sys.stderr.write("! You are missing required software:\n")

if opts.tag and not tt_OK:

sys.stderr.write(" - Tagging is specified but TreeTagger is not installed\n")

if (opts.parse or opts.parse_only or opts.merge_parse) and not malt_OK:

sys.stderr.write(" - Parsing is specified but Malt Parser 1.8 is not installed\n")

response = inp("Attempt to download missing software? [Y/N]\n")

if response.upper().strip() == "Y":

download_requirements(tt_OK,malt_OK)

else:

sys.stderr.write("Aborting\n")

sys.exit(0)

for infile in files:

base = os.path.basename(infile)

if opts.extension is None: # Auto select extension

if opts.parse_only:

opts.extension = "conllu"

else:

opts.extension = "tt"

if opts.outmode == "sgml":

if infile.endswith("." + opts.extension):

outfile = base.replace("." + opts.extension,".out." + opts.extension)

elif len(infile) > 4 and infile[-4] == ".":

outfile = base[:-4] + "." + opts.extension

else:

outfile = base + "." + opts.extension

elif opts.outmode == "conllu":

opts.extension = "conllu"

else:

outfile = base + ".out.txt"

if not opts.quiet:

sys.stderr.write("Processing " + base + "\n")

input_text = io.open(infile,encoding="utf8").read()

if opts.space:

input_text = space_punct(input_text)

if opts.para:

input_text = ekthetic_to_para(input_text)

if opts.merge_parse and not opts.pos_spans:

sys.stderr.write("o --merge_parse was selected; turning on --pos_spans option")

opts.pos_spans = True

processed = nlp_coptic(input_text, lb=opts.breaklines, parse_only=opts.parse_only, do_tok=dotok,

do_norm=opts.norm, do_mwe=opts.multiword, do_tag=opts.tag, do_lemma=opts.lemma,

do_lang=opts.etym, do_milestone=opts.unary, do_parse=opts.parse, sgml_mode=opts.outmode,

tok_mode="auto", old_tokenizer=old_tokenizer, sent_tag=opts.sent, preloaded=stk,

pos_spans=opts.pos_spans, merge_parse=opts.merge_parse, detokenize=opts.detokenize)

if opts.outmode == "sgml":

processed = reorder(processed.strip().split("\n"),add_fixed_meta=add_fixed_meta)

processed = processed.replace("xml:lang","lang")

if len(files) > 1:

with io.open(opts.dirout + os.sep + outfile, 'w', encoding="utf8", newline="\n") as f:

if not PY3:

processed = unicode(processed)

f.write((processed.strip() + "\n"))

else: # Single file, print to stdout

if PY3:

sys.stdout.buffer.write(processed.encode("utf8"))

else:

print(processed.encode("utf8"))

fileword = " files\n\n" if len(files) > 1 else " file\n\n"

sys.stderr.write("\nFinished processing " + str(len(files)) + fileword)