Exemplo n.º 1
0
def uctoTokenizer(input_text: str, output_text: str):
    from ucto import Tokenizer

    """
    A function to create a folia file from a text file

    :param input_text: path to text file
    :param output_text: name for the folia file

    """
    configurationfile = "../ucto_config/tokconfig_eng_ch"

    tokenizer = Tokenizer(configurationfile, foliaoutput=True)

    folia_file_P = Path(input_text)
    isFolder = folia_file_P.is_dir()
    if isFolder:
        # files = [f for f in glob.glob(folia_file + "**/*.xml", recursive=True)]
        files = [f for f in glob.glob(input_text + "**/*.*", recursive=True)]
        pbar = ProgressBar()

        path = Path(output_text)
        if not path.exists():
            path.mkdir()

        for f in pbar(files):
            path_out = f.split("/")
            name_out = path / path_out[len(path_out) - 1]
            out = str(name_out).replace(".txt", ".folia.xml")
            tokenizer.tokenize(f, str(out))
    else:
        tokenizer.tokenize(input_text, output_text)
Exemplo n.º 2
0
import csv
import re
import sys
from lxml import etree
from ucto import Tokenizer

tokenizer = Tokenizer('-L nl -n -Q')
data = etree.parse(sys.argv[1])
with open(sys.argv[1] + ".csv", 'w') as output:
    writer = csv.writer(output, quoting=csv.QUOTE_MINIMAL)
    for row in data.getroot():
        _, idnumber, text, summary = list(row.iterchildren())
        motifs = set()
        if summary.text is None: continue
        for motiflist in re.findall('\[.*?\]', summary.text):
            for motif in re.findall('[^\s\[\],]+', motiflist):
                if not re.search('[0-9]', motif): continue
                motifs.add(motif)
        if motifs:
            print idnumber.text
            text = tokenizer.tokenize(text.text, verbose=False)
            writer.writerow([idnumber.text, r' '.join(motifs), ' '.join(text)])
Exemplo n.º 3
0
import csv
import re
import sys
from lxml import etree
from ucto import Tokenizer

tokenizer = Tokenizer("-L nl -n -Q")
data = etree.parse(sys.argv[1])
with open(sys.argv[1] + ".csv", "w") as output:
    writer = csv.writer(output, quoting=csv.QUOTE_MINIMAL)
    for row in data.getroot():
        _, idnumber, text, summary = list(row.iterchildren())
        motifs = set()
        if summary.text is None:
            continue
        for motiflist in re.findall("\[.*?\]", summary.text):
            for motif in re.findall("[^\s\[\],]+", motiflist):
                if not re.search("[0-9]", motif):
                    continue
                motifs.add(motif)
        if motifs:
            print idnumber.text
            text = tokenizer.tokenize(text.text, verbose=False)
            writer.writerow([idnumber.text, r" ".join(motifs), " ".join(text)])
Exemplo n.º 4
0
    if args.trainingfile:
        if not os.path.isfile(args.trainingfile):
            raise IOError('Trainingfile not found')
        settings['f'] = args.trainingfile
        del settings['i']
    elif args.instancebase:
        if not os.path.isfile(args.instancebase):
            raise IOError('Instancebase not found')
        settings['i'] = args.instancebase

    tokenizer = Tokenizer('-L nl -n -Q')
    with classifier(config.HOST, config.PORT, settings) as program:
        args.output.write(codecs.BOM_UTF8)
        for i, line in enumerate(codecs.open(args.testfile, encoding=config.ENCODING)):
            words = tokenizer.tokenize(line.strip(), tokens = lambda s: s.split())
            output = []
            for word in word:
                results = classifier.classify(word)
                output.append(classifier.pprint_results(results))
            for word, result in zip(words, output):
                args.output.write(
                    u'{0}\t{1}\n'.format(word, result).encode(config.ENCODING))

            if (i+1) % 25 == 0:
                sys.stderr.write(
                    'Processed: {0} words @ {1}\n'.format(i, time.ctime()))