def __init__(self,
nlp,
merge_spans=True,
lookup={},
pattern_id='EMOJI',
attrs=('has_emoji', 'is_emoji', 'emoji_desc', 'emoji')):
"""Initialise the pipeline component.
nlp (Language): The shared nlp object. Used to initialise the matcher
with the shared `Vocab`, and create `Doc` match patterns.
attrs (tuple): Attributes to set on the ._ property. Defaults to
('has_emoji', 'is_emoji', 'emoji_desc', 'emoji').
pattern_id (unicode): ID of match pattern, defaults to 'EMOJI'. Can be
changed to avoid ID clashes.
merge_spans (bool): Merge spans containing multi-character emoji. Will
only merge combined emoji resulting in one icon, not sequences.
lookup (dict): Optional lookup table that maps emoji unicode strings
to custom descriptions, e.g. translations or other annotations.
RETURNS (callable): A spaCy pipeline component.
"""
self._has_emoji, self._is_emoji, self._emoji_desc, self._emoji = attrs
self.merge_spans = merge_spans
self.lookup = lookup
self.matcher = PhraseMatcher(nlp.vocab)
emoji_patterns = [nlp(emoji) for emoji in EMOJI.keys()]
self.matcher.add(pattern_id, None, *emoji_patterns)
# Add attributes
Doc.set_extension(self._has_emoji, getter=self.has_emoji)
Doc.set_extension(self._emoji, getter=self.iter_emoji)
Span.set_extension(self._has_emoji, getter=self.has_emoji)
Span.set_extension(self._emoji, getter=self.iter_emoji)
Token.set_extension(self._is_emoji, default=False)
Token.set_extension(self._emoji_desc, getter=self.get_emoji_desc)
def __init__(self, nlp, label='GPE'):
"""Initialise the pipeline component. The shared nlp instance is used
to initialise the matcher with the shared vocab, get the label ID and
generate Doc objects as phrase match patterns.
"""
# Make request once on initialisation and store the data
r = requests.get('https://restcountries.eu/rest/v2/all')
r.raise_for_status() # make sure requests raises an error if it fails
countries = r.json()
# Convert API response to dict keyed by country name for easy lookup
# This could also be extended using the alternative and foreign language
# names provided by the API
self.countries = {c['name']: c for c in countries}
self.label = nlp.vocab.strings[label] # get entity label ID
# Set up the PhraseMatcher with Doc patterns for each country name
patterns = [nlp(c) for c in self.countries.keys()]
self.matcher = PhraseMatcher(nlp.vocab)
self.matcher.add('COUNTRIES', None, *patterns)
# Register attribute on the Token. We'll be overwriting this based on
# the matches, so we're only setting a default value, not a getter.
# If no default value is set, it defaults to None.
Token.set_extension('is_country', default=False)
Token.set_extension('country_capital')
Token.set_extension('country_latlng')
Token.set_extension('country_flag')
# Register attributes on Doc and Span via a getter that checks if one of
# the contained tokens is set to is_country == True.
Doc.set_extension('has_country', getter=self.has_country)
Span.set_extension('has_country', getter=self.has_country)
def __init__(self, nlp, label='GPE'):
"""Initialise the pipeline component. The shared nlp instance is used
to initialise the matcher with the shared vocab, get the label ID and
generate Doc objects as phrase match patterns.
"""
# Make request once on initialisation and store the data
r = requests.get('https://restcountries.eu/rest/v2/all')
r.raise_for_status() # make sure requests raises an error if it fails
countries = r.json()
# Convert API response to dict keyed by country name for easy lookup
# This could also be extended using the alternative and foreign language
# names provided by the API
self.countries = {c['name']: c for c in countries}
self.label = nlp.vocab.strings[label] # get entity label ID
# Set up the PhraseMatcher with Doc patterns for each country name
patterns = [nlp(c) for c in self.countries.keys()]
self.matcher = PhraseMatcher(nlp.vocab)
self.matcher.add('COUNTRIES', None, *patterns)
# Register attribute on the Token. We'll be overwriting this based on
# the matches, so we're only setting a default value, not a getter.
# If no default value is set, it defaults to None.
Token.set_extension('is_country', default=False)
Token.set_extension('country_capital')
Token.set_extension('country_latlng')
Token.set_extension('country_flag')
# Register attributes on Doc and Span via a getter that checks if one of
# the contained tokens is set to is_country == True.
Doc.set_extension('has_country', getter=self.has_country)
Span.set_extension('has_country', getter=self.has_country)
def add_cat(self, spacy_cat):
self.nlp.add_pipe(spacy_cat, name='cat', last=True)
# Add custom fields needed for this usecase
# Doc.set_extension('ents', default=None, force=True)
Doc.set_extension('tags', default=[], force=True)
Doc.set_extension('tags_as_df', method=_tags_as_dataframe)
def handle_extension_functions_for_token_02(self):
# Register the Doc property extension 'has_number' with the getter get_has_number
Doc.set_extension('has_number', getter=self.__get_has_number__)
# Process the text and check the custom has_number attribute
doc = self.nlp("The museum closed for five years in 2012.")
print('has_number:', doc._.has_number)
def __init__(self, model, word_vocab, char_vocab, chunk_vocab, batch_size: int = 32):
self.model = model
self.bs = batch_size
self.word_vocab = word_vocab
self.char_vocab = char_vocab
self.chunk_vocab = chunk_vocab
Doc.set_extension('noun_phrases', default=[], force=True)
def __init__(self, hunspell_object=None):
self.stopwords_list = [
'the',
'a',
'an',
'are',
'on',
'to',
'at',
'every',
'this'
]
if hunspell_object is None:
hunspell_object = get_hunspell_default()
self.hobj = hunspell_object
self.nlp = spacy.load('en_core_web_lg') # we need another spaCy model as we want to tag the document with
# corrected spelling mistakes
# Todo: maybe always do that at the beginning and just save the
# spelling mistakes
if not Doc.has_extension("lstFilteredDepParseCorpus"):
Doc.set_extension("lstFilteredDepParseCorpus", default=[])
def main():
# Load the small English model
nlp = spacy.load("en_core_web_sm")
# Add the component first in the pipeline and print the pipe names
nlp.add_pipe(length_component, first=True)
print(nlp.pipe_names)
nlp.add_pipe(preprocessor, first=True)
print(nlp.pipe_names)
# Process a text
doc = nlp("This is a sentence.")
# add custom metadata
Doc._.title = 'My document'
Token._.is_color = True
Span._.has_color = False
# set extensions on fht eDoc, Token, and Span
Doc.set_extension('title', default=None)
Token.set_extension('is_color', default=False)
Span.set_extension('has_color', default=False)
# end program
print('Done.')
def __init__(self,
nlp: Language,
matcher: Matcher,
component_name: str,
attribute_name: str,
merge_on_match: bool = True):
"""
Constructor of MatcherComponent object
Args:
nlp (Doc): Spacy Doc object
component_name (str): Unique name of this component
attribute_name (str): label to add for custom attribute
merge_on_match (bool): optional merging on match option, default to True
validate (bool): optional pattern validation, default to False
"""
self.nlp = nlp
self.matcher = matcher
self.name = component_name
self.is_attribute = f"is_{attribute_name}"
self.has_attribute = f"has_{attribute_name}"
self.merge_on_match = merge_on_match
# Register attribute on the Token
Token.set_extension(self.is_attribute, default=False, force=True)
# Register attributes on Doc and Span via a getter function
Doc.set_extension(self.has_attribute,
getter=self.get_attribute,
force=True)
Span.set_extension(self.has_attribute,
getter=self.get_attribute,
force=True)
def __init__(self, nlp: Language):
"""Initialise components"""
extensions = [
"_n_sentences",
"_n_tokens",
"_n_syllables",
"token_length",
"sentence_length",
"syllables",
"counts",
]
ext_funs = [
n_sentences,
n_tokens,
n_syllables,
self.token_length,
self.sentence_length,
self.syllables,
self.counts,
]
for ext, fun in zip(extensions, ext_funs):
if ext not in ["_n_sentences", "sentence_length", "syllables"]:
if not Span.has_extension(ext):
Span.set_extension(ext, getter=fun)
if not Doc.has_extension(ext):
Doc.set_extension(ext, getter=fun)
if not Doc.has_extension("_filtered_tokens"):
Doc.set_extension("_filtered_tokens", default=[])
if not Span.has_extension("_filtered_tokens"):
Span.set_extension("_filtered_tokens", getter=filtered_tokens)
def __init__(self, nlp) -> None:
Doc.set_extension("compound_cases", default=[], force=True)
self.matcher = Matcher(nlp.vocab)
common_pattern = [{
"ent_type": "CASENAME"
}, {
"ent_type": "CITATION",
"OP": "+"
}]
possessive_pattern = [
{
"ent_type": "CASENAME"
},
{
"lower": "case"
},
{
"ent_type": "CITATION"
},
]
self.matcher.add("compound_case", None, common_pattern,
possessive_pattern)
self.global_matcher = Matcher(nlp.vocab)
merge_ents = nlp.create_pipe("merge_entities")
nlp.add_pipe(merge_ents)
def add_meta_cat(self, meta_cat, name):
self.nlp.add_pipe(meta_cat, name=name, last=True)
# Only the meta_anns field is needed, it will be a dictionary
#of {category_name: value, ...}
# Span.set_extension('meta_anns', default=None, force=True)
Doc.set_extension('meta_anns', default=None, force=True)
def __init__(self, nlp) -> None:
Doc.set_extension("abbreviations", default=[], force=True)
Span.set_extension("long_form", default=None, force=True)
self.matcher = Matcher(nlp.vocab)
self.matcher.add("parenthesis", None, [{'ORTH': '('}, {'OP': '+'}, {'ORTH': ')'}])
self.global_matcher = Matcher(nlp.vocab)
def __init__(self, nlp):
"""
Initialise the pipeline component. The shared nlp instance is used
to initialise the matcher with the configured lines (config.ini) and
generate Doc objects as phrase match patterns.
:param nlp: spaCy nlp instance
"""
self.label = nlp.vocab.strings['EVENT'] # get entity label ID
self.matcher = Matcher(nlp.vocab)
self.matcher.add('METRO_DELAYS', None, [{'LOWER': 'retrasos'}], [{'LOWER': 'retraso'}],
[{'LOWER': 'frequencia'}],
[{'LOWER': 'minutos'}, {'LOWER': 'de'}, {'LOWER': 'espera'}],
[{'LOWER': 'min'}, {'LOWER': 'de'}, {'LOWER': 'espera'}],
[{'LOWER': 'minutos'}, {'LOWER': 'de'}, {'LOWER': 'retraso'}],
[{'LOWER': 'min'}, {'LOWER': 'de'}, {'LOWER': 'retraso'}],
[{'LOWER': 'minutos'}, {'LOWER': 'esperando'}],
[{'LOWER': 'tiempo'}, {'LOWER': 'de'}, {'LOWER': 'espera'}],
[{'LOWER': 'tiempos'}, {'LOWER': 'de'}, {'LOWER': 'espera'}],
[{'LOWER': 'frecuencia'}, {'LOWER': 'de'}, {'LOWER': 'paso'}],
[{'LOWER': 'frecuencias'}, {'LOWER': 'de'}, {'LOWER': 'paso'}],
[{'LOWER': 'frecuencias'}, {'LOWER': 'de'}, {'LOWER': 'trenes'}])
# Register attribute on the Token. We'll be overwriting this based on
# the matches, so we're only setting a default value, not a getter.
Token.set_extension('is_metro_delay', default=False)
# Register attributes on Doc and Span via a getter that checks if one of
# the contained tokens is set to is_element_matched == True.
Doc.set_extension('has_metro_delay', getter=self.has_metro_delay)
Span.set_extension('has_metro_delay', getter=self.has_metro_delay)
def __init__(self, nlp):
"""
Initialise the pipeline component. The shared nlp instance is used
to initialise the matcher with the configured lines (config.ini) and
generate Doc objects as phrase match patterns.
:param nlp: spaCy nlp instance
"""
self.label = nlp.vocab.strings['EVENT'] # get entity label ID
self.matcher = Matcher(nlp.vocab)
self.matcher.add('METRO_SOLUTIONS', None, [{'LOWER': 'circulación'}, {'LOWER': 'normalizada'}],
[{'LOWER': 'circulacion'}, {'LOWER': 'normalizada'}],
[{'LOWER': 'servicio'}, {'LOWER': 'normalizado'}],
[{'LOWER': 'normalizado'}, {'LOWER': 'el'}, {'LOWER': 'servicio'}],
[{'LOWER': 'restablecido'}, {'LOWER': 'el'}, {'LOWER': 'servicio'}],
[{'LOWER': 'ya'}, {'LOWER': 'efectúan'}, {'LOWER': 'parada'}],
[{'LOWER': 'ya'}, {'LOWER': 'efectuan'}, {'LOWER': 'parada'}])
# Register attribute on the Token. We'll be overwriting this based on
# the matches, so we're only setting a default value, not a getter.
Token.set_extension('is_metro_solution', default=False)
# Register attributes on Doc and Span via a getter that checks if one of
# the contained tokens is set to is_element_matched == True.
Doc.set_extension('has_metro_solution', getter=self.has_metro_solution)
Span.set_extension('has_metro_solution', getter=self.has_metro_solution)
def __init__(self, attrs=None):
if attrs is None:
self.attrs = (
'n_sents',
'n_words',
'n_chars',
'n_syllables',
'n_unique_words',
'n_long_words',
'n_monosyllable_words',
'n_polysyllable_words',
'flesch_kincaid_grade_level',
'flesch_reading_ease',
'smog_index',
'gunning_fog_index',
'coleman_liau_index',
'automated_readability_index',
'lix',
'gulpease_index',
'wiener_sachtextformel',
)
elif isinstance(attrs, compat.string_types):
self.attrs = (attrs, )
else:
self.attrs = tuple(attrs)
for attr in self.attrs:
SpacyDoc.set_extension(attr, default=None, force=True)
LOGGER.debug('"%s" custom attribute added to `spacy.tokens.Doc`')
def __init__(self, nlp, terminology, label='Match', function=None): # el constructor de nuestro Matcher
self.matcher = Matcher(nlp.vocab) # creamos un objeto Matcher
for topic, patterns in terminology.items(): # cargamos los términos de la mini ontología
for term in patterns:
self.matcher.add(topic, function, term)
Doc.set_extension('rule_match', default=False, force=True) # creamos una extensión al objeto Doc para poder
def __init__(self, attrs=None):
if attrs is None:
self.attrs = (
"n_sents",
"n_words",
"n_chars",
"n_syllables",
"n_unique_words",
"n_long_words",
"n_monosyllable_words",
"n_polysyllable_words",
"flesch_kincaid_grade_level",
"flesch_reading_ease",
"smog_index",
"gunning_fog_index",
"coleman_liau_index",
"automated_readability_index",
"lix",
"gulpease_index",
"wiener_sachtextformel",
)
elif isinstance(attrs, (str, bytes)):
self.attrs = (attrs,)
else:
self.attrs = tuple(attrs)
for attr in self.attrs:
# TODO: see if there's a better way to handle this
# that doesn't involve clobbering existing property extensions
Doc.set_extension(attr, default=None, force=True)
LOGGER.debug('"%s" custom attribute added to `spacy.tokens.Doc`')
def __init__(self, nlp) -> None:
Doc.set_extension("concepts", default={}, force=True)
self.ruler = EntityRuler(nlp)
self.ruler.add_patterns(CONCEPT_PATTERNS)
pipes = nlp.pipe_names
if "EntityRuler" not in pipes:
nlp.add_pipe(self.ruler, last=True)
def __init__(self, attrs: Optional[Union[str, Collection[str]]] = None):
self._set_attrs(attrs)
for attr in self.attrs:
# TODO: see if there's a better way to handle this
# that doesn't involve clobbering existing property extensions
Doc.set_extension(attr, default=None, force=True)
LOGGER.debug('"%s" custom attribute added to `spacy.tokens.Doc`')
def __init__(self, nlp, label="PRODUCT"):
labels = []
APP_ROOT = os.path.dirname(os.path.abspath(__file__))
with open(os.path.join(APP_ROOT, 'labels.json')) as f:
labels = json.loads(f.read())
self.labels = { c["name"].lower(): c for c in labels}
self.label = nlp.vocab.strings[label] # get entity label ID
patterns = [nlp(c) for c in self.labels.keys()]
self.matcher = PhraseMatcher(nlp.vocab,attr='LOWER')
self.matcher.add("PRODUCTS", None, *patterns)
# Register attribute on the Token. We'll be overwriting this based on the matches
Token.set_extension("is_product", default=False, force=True)
Token.set_extension("type", default=False, force=True)
# Register attributes on Doc and Span via a getter that checks if one of
# the contained tokens is set to is_product== True.
Doc.set_extension("has_product", getter=self.has_product, force=True)
Span.set_extension("has_product", getter=self.has_product, force=True)
def __init__(self, nlp, companies=tuple(), label='PERSON'):
"""Initialise the pipeline component. The shared nlp instance is used
to initialise the matcher with the shared vocab, get the label ID and
generate Doc objects as phrase match patterns.
"""
self.label = nlp.vocab.strings[label] # get entity label ID
# Set up the PhraseMatcher – it can now take Doc objects as patterns,
# so even if the list of companies is long, it's very efficient
patterns = [nlp(org) for org in companies]
self.matcher = PhraseMatcher(nlp.vocab)
self.matcher.add('DANISH_NAMES', None, *patterns)
# Register attribute on the Token. We'll be overwriting this based on
# the matches, so we're only setting a default value, not a getter.
Token.set_extension('is_danish_name', default=False)
# Register attributes on Doc and Span via a getter that checks if one of
# the contained tokens is set to is_tech_org == True.
Doc.set_extension('has_danish_name', getter=self.has_tech_org)
Span.set_extension('has_danish_name', getter=self.has_tech_org)
def __init__(self, nlp, lei, label="LEI"):
"""Initialise the pipeline component. The shared nlp instance is used
to initialise the ruler with the shared vocab, generate Doc objects as
phrase match patterns.
"""
# Make request once on initialisation and store the data
self.label = label
# Set up the EntityRuler
patterns = [{
"label":
self.label,
"pattern": [{
"LOWER": r.lower()
} for r in result["LegalName"].split()]
} for result in lei["records"]]
self.ruler = EntityRuler(nlp, overwrite_ents=True)
self.ruler.add_patterns(patterns=patterns)
# Register attribute on the Token with default value.
# I will overwrite this based on the matches.
Token.set_extension("is_lei", default=False)
# Register attributes on Doc and Span via a getter that checks if one of
# the contained tokens is set to is_lei == True.
Doc.set_extension("has_lei", getter=self.has_lei)
Span.set_extension("has_lei", getter=self.has_lei)
def _set_spacy_extensions(self):
def synset_getter(token):
if not Disambiguator._wn:
from nltk.corpus import wordnet as wn
Disambiguator._wn = wn
else:
wn = Disambiguator._wn
offset = token._.offset
if offset:
return wn.synset_from_pos_and_offset(offset[-1],
int(offset[3:-1]))
else:
return None
def bnid_getter(token):
return babelnet_map.get(token._.offset)
from spacy.tokens import Doc, Token
Doc.set_extension('lang', default='en')
Token.set_extension('lemma_preset_', default=None)
Token.set_extension('pos_preset_', default=None)
Token.set_extension('lemma_preset_else_spacy',
getter=lambda t: t._.lemma_preset_ or t.lemma_)
Token.set_extension('pos_preset_else_spacy',
getter=lambda t: t._.pos_preset_ or t.pos_)
Token.set_extension('offset', default=None)
Token.set_extension('synset', getter=synset_getter)
Token.set_extension('bnid', getter=bnid_getter)
Token.set_extension('disambiguator_internals', default=None)
def __call__(
self,
doc: Doc,
) -> Doc:
"""
Set the extension attributes on a `spaCy` [`Doc`](https://spacy.io/api/doc)
document to create a *pipeline component* for `TextRank` as
a stateful component, invoked when the document gets processed.
See: <https://spacy.io/usage/processing-pipelines#pipelines>
doc:
a document container, providing the annotations produced by earlier stages of the `spaCy` pipeline
"""
Doc.set_extension("textrank", force=True, default=None)
Doc.set_extension("phrases", force=True, default=[])
doc._.textrank = BaseTextRank(
doc,
edge_weight=self.edge_weight,
pos_kept=self.pos_kept,
token_lookback=self.token_lookback,
scrubber=self.scrubber,
stopwords=self.stopwords,
)
doc._.phrases = doc._.textrank.calc_textrank()
return doc
def __init__(self,
*,
from_pretrained: Dict[LANG_ISO_639_1, str],
attr_name: str = 'bert_repr',
max_seq_len: int = 512,
pooling_strategy: str = 'REDUCE_MEAN',
set_extension: bool = True,
force_extension: bool = True):
"""
Use with spacy pipeline for getting BERT (PyTorch) language-specific
tensor representations when multiple languages are present in the
dataset and the languages are pre-known.
Use after applying spacy_langdetect.LanguageDetector() (in pipeline)
# https://spacy.io/universe/project/spacy-langdetect
Keyword arguments only!
Params
------
from_pretrained: Dict[LANG_ISO_639_1, str]
Mapping between two-letter language codes to path to model
directory or HuggingFace transformers pre-trained Bert weights.
attr_name: str (default='bert_repr')
Same as in BertInference.
max_seq_len: int (default=512)
Same as in BertInference.
pooling_strategy: str (default='REDUCE_MEAN')
Same as in BertInference.
set_extension: bool (default=True)
Same as in BertInference.
force_extension: bool (default=True)
Same as in BertInference.
"""
self.from_pretrained = from_pretrained
self.attr_name = attr_name
self.max_seq_len = max_seq_len
self.pooling_strategy = pooling_strategy
self.set_extension = set_extension
self.force_extension = force_extension
if set_extension:
Doc.set_extension(attr_name,
getter=self.__call__,
force=force_extension)
Span.set_extension(attr_name,
getter=self.__call__,
force=force_extension)
Token.set_extension(attr_name,
getter=self.__call__,
force=force_extension)
else:
Doc.set_extension(attr_name, default=None, force=force_extension)
Span.set_extension(attr_name, default=None, force=force_extension)
Token.set_extension(attr_name, default=None, force=force_extension)
def __init__(self, commoditiesIdNm={}, commoditiesIdCode={}):
# self.countriesIdNm = countriesIdNm if len(countriesIdNm) else None
self.commoditiesIdNm = commoditiesIdNm if len(commoditiesIdNm) else None
self.commoditiesIdCode = commoditiesIdCode if len(commoditiesIdCode) else None
########## stop words
self.commStopWords = {
0: reCompile("partner|independ|executive|director|trade|negotiator"\
+"|role|auditor|declar|represent|manager|offer|engage|long|time"\
+"|underwriter|party|integrated|business|sponsor|joint|role"\
+"|co-lead|agents|assignment|arranger|plaintiff|lender|engineer"\
+"|player|dealer|private|rich|corporate|communications|email"\
+"|advisor|arranger|regulation|estimator|consultant|up", reIGNORECASE )
}
########## Commodities
if self.commoditiesIdNm is not None:
comPat = ''
for commid, names in self.commoditiesIdNm.items():
namespat = '\W|\W'.join( names )
comPat = comPat + f"|(?P<com{commid}>\\W{namespat}\\W)"
comPat = comPat.strip('|')
comCodePat = ''
for commid, commcode in self.commoditiesIdCode.items():
comCodePat = comCodePat + f"|(?P<com{commid}>\\W{commcode}\\W)"
comCodePat = comCodePat.strip('|')
self.comCodePatCompiled = reCompile(comCodePat)
self.comPatCompiled = reCompile(comPat, reIGNORECASE)
Span.set_extension('commodities', getter=self.Commodities, force=True)
Doc.set_extension('commodities', default=[], force=True)
def __init__(self,
nlp: Language,
name: str = "hyponym_detector",
extended: bool = False):
self.nlp = nlp
self.patterns = BASE_PATTERNS
if extended:
self.patterns.extend(EXTENDED_PATTERNS)
self.matcher = Matcher(self.nlp.vocab)
Doc.set_extension("hearst_patterns", default=[], force=True)
self.first = set()
self.last = set()
# add patterns to matcher
for pattern in self.patterns:
self.matcher.add(pattern["label"], [pattern["pattern"]])
# gather list of predicates where the hypernym appears first
if pattern["position"] == "first":
self.first.add(pattern["label"])
# gather list of predicates where the hypernym appears last
if pattern["position"] == "last":
self.last.add(pattern["label"])
def __init__(self, nlp):
"""
Initialise the pipeline component. The shared nlp instance is used
to initialise the matcher with the configured lines (config.ini) and
generate Doc objects as phrase match patterns.
:param nlp: spaCy nlp instance
"""
self.label = nlp.vocab.strings['EVENT'] # get entity label ID
self.matcher = Matcher(nlp.vocab)
self.matcher.add('METRO_STRIKES', None, [{'LOWER': 'huelga'}], [{'LOWER': 'paros'}],
[{'LOWER': 'servicios'}, {'LOWER': 'minimos'}],
[{'LOWER': 'servicios'}, {'LOWER': 'mínimos'}],
[{'LOWER': 'paros'}, {'LOWER': 'convocados'}, {'LOWER': 'para'}, {'LOWER': 'hoy'}],
[{'LOWER': 'paros'}, {'LOWER': 'convocados'}, {'LOWER': 'para'}, {'LOWER': 'mañana'}],
[{'LOWER': 'paros'}, {'LOWER': 'convocados'}, {'LOWER': 'para'}, {'LOWER': 'el'},
{'LOWER': 'dia'}, {'IS_DIGIT': True}],
[{'LOWER': 'paros'}, {'LOWER': 'convocados'}, {'LOWER': 'para'}, {'LOWER': 'el'},
{'LOWER': 'día'}, {'IS_DIGIT': True}],
[{'LOWER': 'paros'}, {'LOWER': 'convocados'}])
# Register attribute on the Token. We'll be overwriting this based on
# the matches, so we're only setting a default value, not a getter.
Token.set_extension('is_metro_strike', default=False)
# Register attributes on Doc and Span via a getter that checks if one of
# the contained tokens is set to is_element_matched == True.
Doc.set_extension('has_metro_strike', getter=self.has_metro_strike)
Span.set_extension('has_metro_strike', getter=self.has_metro_strike)
def __init__(self, nlp):
"""
Initialise the pipeline component. The shared nlp instance is used
to initialise the matcher with the configured stations (config.ini) and
generate Doc objects as phrase match patterns.
:param nlp: spaCy nlp instance
"""
self.label = nlp.vocab.strings['FACILITY'] # get entity label ID
# Set up the PhraseMatcher – it can now take Doc objects as patterns,
# so even if the list of companies is long, it's very efficient
metro_stations = config['keywords']['stations'].split(',')
metro_stations += config['keywords']['stations_lw'].split(',')
patterns = [nlp(org) for org in metro_stations]
self.matcher = PhraseMatcher(nlp.vocab)
self.matcher.add('METRO_STATIONS', None, *patterns)
# Register attribute on the Token. We'll be overwriting this based on
# the matches, so we're only setting a default value, not a getter.
Token.set_extension('is_metro_station', default=False)
# Register attributes on Doc and Span via a getter that checks if one of
# the contained tokens is set to is_element_matched == True.
Doc.set_extension('has_metro_station', getter=self.has_metro_station)
Span.set_extension('has_metro_station', getter=self.has_metro_station)
def __init__(self,
nlp,
func: Callable[..., pd.Series] = combo_basic,
force: bool = True,
*args,
**kwargs) -> None:
"""
This is for initializing the TermExtractionPipeline.
"""
self.func = func
self.args = args
self.kwargs = kwargs
self.__name__ = self.func.__name__
self.matcher = Matcher(nlp.vocab)
Doc.set_extension(self.__name__, default=None, force=force)
self.term_counter = None
def add_to_counter(matcher, doc, i, matches) -> Doc:
match_id, start, end = matches[i]
candidate = str(doc[start:end])
if (TermExtraction.word_length(candidate) <=
TermExtraction.config["MAX_WORD_LENGTH"]):
self.term_counter[candidate] += 1
for i, pattern in enumerate(TermExtraction.patterns):
self.matcher.add("term{}".format(i), add_to_counter, pattern)
def test_doc_to_json_underscore(doc):
Doc.set_extension("json_test1", default=False)
Doc.set_extension("json_test2", default=False)
doc._.json_test1 = "hello world"
doc._.json_test2 = [1, 2, 3]
json_doc = doc.to_json(underscore=["json_test1", "json_test2"])
assert "_" in json_doc
assert json_doc["_"]["json_test1"] == "hello world"
assert json_doc["_"]["json_test2"] == [1, 2, 3]
def test_underscore_dir(en_vocab):
"""Test that dir() correctly returns extension attributes. This enables
things like tab-completion for the attributes in doc._."""
Doc.set_extension("test_dir", default=None)
doc = Doc(en_vocab, words=["hello", "world"])
assert "_" in dir(doc)
assert "test_dir" in dir(doc._)
assert "test_dir" not in dir(doc[0]._)
assert "test_dir" not in dir(doc[0:2]._)
def test_underscore_docstring(en_vocab):
"""Test that docstrings are available for extension methods, even though
they're partials."""
def test_method(doc, arg1=1, arg2=2):
"""I am a docstring"""
return (arg1, arg2)
Doc.set_extension("test_docstrings", method=test_method)
doc = Doc(en_vocab, words=["hello", "world"])
assert test_method.__doc__ == "I am a docstring"
assert doc._.test_docstrings.__doc__.rsplit(". ")[-1] == "I am a docstring"
def test_underscore_mutable_defaults_list(en_vocab):
"""Test that mutable default arguments are handled correctly (see #2581)."""
Doc.set_extension("mutable", default=[])
doc1 = Doc(en_vocab, words=["one"])
doc2 = Doc(en_vocab, words=["two"])
doc1._.mutable.append("foo")
assert len(doc1._.mutable) == 1
assert doc1._.mutable[0] == "foo"
assert len(doc2._.mutable) == 0
doc1._.mutable = ["bar", "baz"]
doc1._.mutable.append("foo")
assert len(doc1._.mutable) == 3
assert len(doc2._.mutable) == 0
def main(output_dir=None):
nlp = English() # start off with blank English class
Doc.set_extension('overlap', method=overlap_tokens)
doc1 = nlp(u"Peach emoji is where it has always been.")
doc2 = nlp(u"Peach is the superior emoji.")
print("Text 1:", doc1.text)
print("Text 2:", doc2.text)
print("Overlapping tokens:", doc1._.overlap(doc2))
Doc.set_extension('to_html', method=to_html)
doc = nlp(u"This is a sentence about Apple.")
# add entity manually for demo purposes, to make it work without a model
doc.ents = [Span(doc, 5, 6, label=nlp.vocab.strings['ORG'])]
print("Text:", doc.text)
doc._.to_html(output=output_dir, style='ent')
def __init__(self, nlp, companies=tuple(), label='ORG'):
"""Initialise the pipeline component. The shared nlp instance is used
to initialise the matcher with the shared vocab, get the label ID and
generate Doc objects as phrase match patterns.
"""
self.label = nlp.vocab.strings[label] # get entity label ID
# Set up the PhraseMatcher – it can now take Doc objects as patterns,
# so even if the list of companies is long, it's very efficient
patterns = [nlp(org) for org in companies]
self.matcher = PhraseMatcher(nlp.vocab)
self.matcher.add('TECH_ORGS', None, *patterns)
# Register attribute on the Token. We'll be overwriting this based on
# the matches, so we're only setting a default value, not a getter.
Token.set_extension('is_tech_org', default=False)
# Register attributes on Doc and Span via a getter that checks if one of
# the contained tokens is set to is_tech_org == True.
Doc.set_extension('has_tech_org', getter=self.has_tech_org)
Span.set_extension('has_tech_org', getter=self.has_tech_org)
def doc_w_attrs(en_tokenizer):
Doc.set_extension("_test_attr", default=False)
Doc.set_extension("_test_prop", getter=lambda doc: len(doc.text))
Doc.set_extension(
"_test_method", method=lambda doc, arg: "{}{}".format(len(doc.text), arg)
)
doc = en_tokenizer("This is a test.")
doc._._test_attr = "test"
return doc
def test_underscore_accepts_valid(valid_kwargs):
valid_kwargs["force"] = True
Doc.set_extension("test", **valid_kwargs)
def test_underscore_raises_for_invalid(invalid_kwargs):
invalid_kwargs["force"] = True
with pytest.raises(ValueError):
Doc.set_extension("test", **invalid_kwargs)
def test_doc_to_json_underscore_error_serialize(doc):
"""Test that Doc.to_json() raises an error if a custom attribute value
isn't JSON-serializable."""
Doc.set_extension("json_test4", method=lambda doc: doc.text)
with pytest.raises(ValueError):
doc.to_json(underscore=["json_test4"])