# encoding: utf8 from __future__ import unicode_literals from spacy.language_data import strings_to_exc, update_exc from .punctuation import * from .stop_words import STOP_WORDS from .tag_map import TAG_MAP as TAG_MAP_BN from .morph_rules import MORPH_RULES from .lemma_rules import LEMMA_RULES from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS as TOKENIZER_EXCEPTIONS_BN from .. import language_data as base STOP_WORDS = set(STOP_WORDS) TAG_MAP = base.TAG_MAP TAG_MAP.update(TAG_MAP_BN) TOKENIZER_EXCEPTIONS = strings_to_exc(base.EMOTICONS) update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(base.ABBREVIATIONS)) TOKENIZER_EXCEPTIONS.update(TOKENIZER_EXCEPTIONS_BN) TOKENIZER_PREFIXES = TOKENIZER_PREFIXES TOKENIZER_SUFFIXES = TOKENIZER_SUFFIXES TOKENIZER_INFIXES = TOKENIZER_INFIXES __all__ = ["TOKENIZER_EXCEPTIONS", "STOP_WORDS", "TAG_MAP", "MORPH_RULES", "LEMMA_RULES", "TOKENIZER_PREFIXES", "TOKENIZER_SUFFIXES", "TOKENIZER_INFIXES"]
# coding: utf8 from __future__ import unicode_literals import spacy from spacy import language_data as base from spacy.language_data import update_exc, strings_to_exc, expand_exc from spacy.en.word_sets import NUM_WORDS from spacy.en.tokenizer_exceptions import ORTH_ONLY from allennlp.data.tokenizers.tokenizer_exceptions import TOKENIZER_EXCEPTIONS TOKENIZER_EXCEPTIONS = dict(TOKENIZER_EXCEPTIONS) update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(ORTH_ONLY)) update_exc(TOKENIZER_EXCEPTIONS, expand_exc(TOKENIZER_EXCEPTIONS, "'", "’")) update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(base.EMOTICONS)) update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(base.ABBREVIATIONS))
# coding: utf8 from __future__ import unicode_literals from spacy.language_data import strings_to_exc, update_exc from .punctuation import * from .stop_words import STOP_WORDS from .tag_map import TAG_MAP as TAG_MAP_BN from .morph_rules import MORPH_RULES from .lemma_rules import LEMMA_RULES from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS as TOKENIZER_EXCEPTIONS_BN from .. import language_data as base STOP_WORDS = set(STOP_WORDS) TAG_MAP = base.TAG_MAP TAG_MAP.update(TAG_MAP_BN) TOKENIZER_EXCEPTIONS = strings_to_exc(base.EMOTICONS) update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(base.ABBREVIATIONS)) TOKENIZER_EXCEPTIONS.update(TOKENIZER_EXCEPTIONS_BN) TOKENIZER_PREFIXES = TOKENIZER_PREFIXES TOKENIZER_SUFFIXES = TOKENIZER_SUFFIXES TOKENIZER_INFIXES = TOKENIZER_INFIXES __all__ = ["TOKENIZER_EXCEPTIONS", "STOP_WORDS", "TAG_MAP", "MORPH_RULES", "LEMMA_RULES", "TOKENIZER_PREFIXES", "TOKENIZER_SUFFIXES", "TOKENIZER_INFIXES"]
# encoding: utf8 from __future__ import unicode_literals import six from spacy.language_data import strings_to_exc, update_exc from .punctuations import * from .stop_words import STOP_WORDS from .tokenizer_exceptions import ABBREVIATIONS from .tokenizer_exceptions import OTHER_EXC from .. import language_data as base STOP_WORDS = set(STOP_WORDS) TOKENIZER_EXCEPTIONS = strings_to_exc(base.EMOTICONS) TOKENIZER_PREFIXES = base.TOKENIZER_PREFIXES + TOKENIZER_PREFIXES TOKENIZER_SUFFIXES = TOKENIZER_SUFFIXES TOKENIZER_INFIXES = TOKENIZER_INFIXES # HYPHENS = [six.unichr(cp) for cp in [173, 8211, 8212, 8213, 8722, 9472]] update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(OTHER_EXC)) update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(ABBREVIATIONS)) __all__ = [ "TOKENIZER_EXCEPTIONS", "STOP_WORDS", "TOKENIZER_PREFIXES", "TOKENIZER_SUFFIXES", "TOKENIZER_INFIXES" ]