Exemplos de Tokenizer.HuTokenizer em Python

Exemplo n.º 1

Arquivo: Playground.py Projeto: Prodinal/GateSpacyWrapping

def create_spacy_hu():
    nlp = spacy.blank('hu')
    nlp.tokenizer = Tokenizer.HuTokenizer(nlp.vocab)

    morph_analyzer = LemmatizerMorphAnalyzer.HuLemmaMorph(nlp)
    nlp.add_pipe(morph_analyzer)

    constitutency_parser = ConstitutencyParser.ConstitutencyParser(nlp)
    nlp.add_pipe(constitutency_parser)

    dependency_parser = DependencyParser.DependencyParser(nlp)
    nlp.add_pipe(dependency_parser)

    np_chunker = NPChunker.NPChunker(nlp)
    nlp.add_pipe(np_chunker)

    POS_analyzer = POSTagger.HuPOSTagger(nlp)
    nlp.add_pipe(POS_analyzer)

    preverb_identifier = PreverbIdentifier.PreverbIdentifier(nlp)
    nlp.add_pipe(preverb_identifier)

    hu_word_to_vec = HuWordToVec.HUWordToVec()
    nlp.add_pipe(hu_word_to_vec)

    return nlp

Exemplo n.º 2

    def __init__(self):
        nlp = spacy.blank('hu')
        nlp.tokenizer = Tokenizer.HuTokenizer(nlp.vocab)

        morph_analyzer = LemmatizerMorphAnalyzer.HuLemmaMorph(nlp)
        nlp.add_pipe(morph_analyzer)

        constitutency_parser = ConstitutencyParser.ConstitutencyParser(nlp)
        nlp.add_pipe(constitutency_parser)

        dependency_parser = DependencyParser.DependencyParser(nlp)
        nlp.add_pipe(dependency_parser)

        np_chunker = NPChunker.NPChunker(nlp)
        nlp.add_pipe(np_chunker)

        POS_analyzer = POSTagger.HuPOSTagger(nlp)
        nlp.add_pipe(POS_analyzer)

        preverb_identifier = PreverbIdentifier.PreverbIdentifier(nlp)
        nlp.add_pipe(preverb_identifier)

        self.nlp = nlp