def __call__(self, doc: Doc):
save_parsed = doc.is_parsed
doc.is_parsed = False
if self.split_matcher:
matches = self.split_matcher(doc)
for match_id, start, end in matches:
token = doc[end - 1]
token.is_sent_start = True
if end - 2 >= 0 and doc[end - 2].is_sent_start is True:
doc[end - 2].is_sent_start = False
if self.join_matcher:
matches = self.join_matcher(doc)
for match_id, start, end in matches:
# If there is a sent start in the match, just remove it
for token in doc[start:end]:
if token.is_sent_start:
token.is_sent_start = False
if doc.is_sentenced:
# Trim starting spaces
sent_start = None
for sent in doc.sents:
sentlen = len(sent)
first_non_space = 0
while first_non_space < sentlen and sent[
first_non_space].is_space:
first_non_space += 1
if first_non_space > 0 and first_non_space < sentlen:
sent[0].is_sent_start = False
sent[first_non_space].is_sent_start = True
doc.is_parsed = save_parsed if doc.is_sentenced else True
return doc
def handle(self, *args, **options):
spacy_model = spacy.load(settings.SPACY_CUSTOMN_MODEL_FOLDER, disable=['parser', 'tagger', 'textcat'])
Span.set_extension('is_phone', getter=Command.is_phone_getter, force=True)
Span.set_extension('line_number', getter=Command.line_number_getter, force=True)
Doc.set_extension('lines', getter=Command.get_lines, setter=Command.set_lines)
Doc.set_extension('_lines', default=list())
logger.debug("Loaded spacy server")
main_socks, read_socks, write_socks = socket_bind('', settings.SPACY_PORT)
while True:
readable, writeable, exceptions = select(read_socks, write_socks, [])
for sockobj in readable:
if sockobj in main_socks:
new_sock, address = sockobj.accept()
logger.debug('Connect: %s - %s', address, id(new_sock))
read_socks.append(new_sock)
else:
try:
entities = []
data = recv_end(sockobj)
if not data:
sockobj.close()
read_socks.remove(sockobj)
else:
for doc in spacy_model.pipe([data]):
doc._.lines = [x.start() for x in re.finditer('\n', doc.text)]
for ent in doc.ents:
current_entity = self.get_ent(ent)
entities.append(current_entity) if current_entity else None
sockobj.sendall(json.dumps(entities).encode('utf8') + '--end--'.encode('utf8'))
except:
pass
def test_match_zero(matcher):
matcher.add('Quote', '', {}, [[{
'ORTH': '"'
}, {
'OP': '!',
'IS_PUNCT': True
}, {
'OP': '!',
'IS_PUNCT': True
}, {
'ORTH': '"'
}]])
doc = Doc(matcher.vocab, words='He said , " some words " ...'.split())
assert len(matcher(doc)) == 1
doc = Doc(matcher.vocab,
words='He said , " some three words " ...'.split())
assert len(matcher(doc)) == 0
matcher.add('Quote', '', {}, [[{
'ORTH': '"'
}, {
'IS_PUNCT': True
}, {
'IS_PUNCT': True
}, {
'IS_PUNCT': True
}, {
'ORTH': '"'
}]])
assert len(matcher(doc)) == 0
def _make_span(self, doc: Doc, start: int, end: int, label: str,
is_char: bool, retok: bool):
span: Span
if is_char:
if label is None:
span = doc.char_span(start, end)
else:
span = doc.char_span(start, end, label=label)
else:
if label is None:
span = Span(doc, start, end)
else:
span = Span(doc, start, end, label=label)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(f'span ({start}, {end}) for {label}: {span}')
if span is not None:
# this is a span object or none if match doesn't map to valid token
# sequence
if logger.isEnabledFor(logging.DEBUG):
logger.debug(f'match: {span.text}')
if label is not None:
doc.ents += (span, )
if retok:
# https://github.com/explosion/spaCy/discussions/4806
with doc.retokenize() as retokenizer:
# Iterate over all spans and merge them into one
# token. This is done after setting the entities –
# otherwise, it would cause mismatched indices!
retokenizer.merge(span)
def __init__(self,
nlp: Language = None,
support_overlap: bool = False,
log_level: int = logging.WARNING,
encoding: str = None,
doc_name_depth: int = 0,
**kwargs):
"""
@param nlp: Spacy Language model
@param support_overlap: whether need to support overlapped annotations
@param log_level: logging level configuration
@param encoding: txt encoding
@param doc_name_depth: depth of parent directories to add into doc_name
default is 0: only use file name
1: use 1 level parent directory name + file name
-1: use full absolution path
if you are dealing with multiple directories,this is helpful to
locate the original files
@param kwargs:other parameters
"""
for param_name, value in kwargs.items():
setattr(self, param_name, value)
if nlp is None:
raise NameError('parameter "nlp" need to be defined')
self.nlp = nlp
self.encoding = encoding
self.doc_name_depth = doc_name_depth
self.support_overlap = support_overlap
self.set_logger(log_level)
if not Doc.has_extension('doc_name'):
Doc.set_extension('doc_name', default='')
pass
def get_spacy(self,load_from_file=False,model_name='en_core_web_sm'):
import spacy
global nlp
if not nlp:
#print('>> loading spacy...')
nlp = spacy.load(model_name)
doc=None
if self.parsed and load_from_file:
#print self.fnfn_spacy
from spacy.tokens.doc import Doc
try:
for byte_string in Doc.read_bytes(open(self.fnfn_spacy, 'rb')):
doc = Doc(nlp.vocab)
doc.from_bytes(byte_string)
except UnicodeDecodeError:
print("!! UNICODE ERROR:",self.fnfn_spacy)
#else:
if not doc:
#print '>> making spacy document for text',self.id
txt=self.text
txt=clean_text(txt)
doc=nlp(txt)
return doc
def __init__(self):
super().__init__()
if not Doc.has_extension(self.name):
Doc.set_extension(self.name, default=[])
if not Token.has_extension('is_lexical'):
Token.set_extension('is_lexical', default=False)
def add_span_extensions():
Doc.set_extension("relations", default=None)
Doc.set_extension("entities", default=None)
for span_extension in [
'entity_type', 'entity_id', 'foodon', 'hansard', 'hansardClosest',
'hansardParent', 'snomedct', 'synonyms'
]:
Span.set_extension(span_extension, default=None)
def make_doc_from_text_chunks(text, lang, chunk_size=100000):
"""
Make a single spaCy-processed document from 1 or more chunks of ``text``.
This is a workaround for processing very long texts, for which spaCy
is unable to allocate enough RAM.
Although this function's performance is *pretty good*, it's inherently
less performant that just processing the entire text in one shot.
Only use it if necessary!
Args:
text (str): Text document to be chunked and processed by spaCy.
lang (str or ``spacy.Language``): A 2-letter language code (e.g. "en"),
the name of a spaCy model for the desired language, or
an already-instantiated spaCy language pipeline.
chunk_size (int): Number of characters comprising each text chunk
(excluding the last chunk, which is probably smaller). For best
performance, value should be somewhere between 1e3 and 1e7,
depending on how much RAM you have available.
.. note:: Since chunking is done by character, chunks edges' probably
won't respect natural language segmentation, which means that every
``chunk_size`` characters, spaCy will probably get tripped up and
make weird parsing errors.
Returns:
``spacy.Doc``: A single processed document, initialized from
components accumulated chunk by chunk.
"""
if isinstance(lang, compat.unicode_):
lang = cache.load_spacy(lang)
elif not isinstance(lang, SpacyLang):
raise TypeError('`lang` must be {}, not {}'.format(
{compat.unicode_, SpacyLang}, type(lang)))
words = []
spaces = []
np_arrays = []
cols = [
attrs.POS, attrs.TAG, attrs.DEP, attrs.HEAD, attrs.ENT_IOB,
attrs.ENT_TYPE
]
text_len = len(text)
i = 0
# iterate over text chunks and accumulate components needed to make a doc
while i < text_len:
chunk_doc = lang(text[i:i + chunk_size])
words.extend(tok.text for tok in chunk_doc)
spaces.extend(bool(tok.whitespace_) for tok in chunk_doc)
np_arrays.append(chunk_doc.to_array(cols))
i += chunk_size
# now, initialize the doc from words and spaces
# then load attribute values from the concatenated np array
doc = SpacyDoc(lang.vocab, words=words, spaces=spaces)
doc = doc.from_array(cols, np.concatenate(np_arrays, axis=0))
return doc
def __init__(self, clf, extension='score'):
"""
:type clf: Classifier, needs to have a predict(X) function
"""
self.clf = clf
self.extension = extension
if not Doc.has_extension(extension):
Doc.set_extension(extension, default=-1)
def read_docs(filepath):
"""Deserialize a list of documents + associated metadata"""
spacy_parser = get_spacy_parser()
data = pickle.load(open(filepath, 'rb'))
for row in data:
doc = Doc(spacy_parser.vocab)
# read doc object from serialized byte array
row['content'] = doc.from_bytes(row.pop('binary_content'))
return data
def test_efficient_binary_serialization(doc):
from spacy.tokens.doc import Doc
byte_string = doc.to_bytes()
open('moby_dick.bin', 'wb').write(byte_string)
nlp = spacy.en.English()
for byte_string in Doc.read_bytes(open('moby_dick.bin', 'rb')):
doc = Doc(nlp.vocab)
doc.from_bytes(byte_string)
def test_read_bytes(nlp):
from spacy.tokens.doc import Doc
loc = 'test_serialize.bin'
with open(loc, 'wb') as file_:
file_.write(nlp(u'This is a document.').to_bytes())
file_.write(nlp(u'This is another.').to_bytes())
docs = []
with open(loc, 'rb') as file_:
for byte_string in Doc.read_bytes(file_):
docs.append(Doc(nlp.vocab).from_bytes(byte_string))
assert len(docs) == 2
def deserialize_dataset(file_path, max_items):
vocab = spacy.load('en_default').vocab
with open(file_path, 'rb') as input_file:
for data_point in pickle.load(input_file)[:max_items]:
yield {
'question1': Doc(vocab).from_bytes(data_point['question1']),
'question2': Doc(vocab).from_bytes(data_point['question2']),
'id': data_point['id'],
'is_duplicate': data_point.get('is_duplicate', None)
}
def load_and_transform(batch_id, in_loc, out_dir):
out_loc = path.join(out_dir, '%d.txt' % batch_id)
if path.exists(out_loc):
return None
print('Batch', batch_id)
nlp = spacy.en.English(parser=False, tagger=False, matcher=False, entity=False)
with io.open(out_loc, 'w', encoding='utf8') as out_file:
with io.open(in_loc, 'rb') as in_file:
for byte_string in Doc.read_bytes(in_file):
doc = Doc(nlp.vocab).from_bytes(byte_string)
doc.is_parsed = True
out_file.write(transform_doc(doc))
def load_and_transform(batch_id, in_loc, out_dir):
out_loc = path.join(out_dir, '%d.txt' % batch_id)
if path.exists(out_loc):
return None
print('Batch', batch_id)
nlp = spacy.en.English(parser=False, tagger=False, matcher=False, entity=False)
with io.open(out_loc, 'w', encoding='utf8') as out_file:
with io.open(in_loc, 'rb') as in_file:
for byte_string in Doc.read_bytes(in_file):
doc = Doc(nlp.vocab).from_bytes(byte_string)
doc.is_parsed = True
out_file.write(transform_doc(doc))
def test_docs_to_sents_df(self):
if Doc.has_extension("concepts"):
Doc.remove_extension("concepts")
dir_reader = EhostDirReader(nlp=self.nlp, support_overlap=False,
recursive=True,
schema_file='data/ehost_test_corpus/config/projectschema.xml')
docs = dir_reader.read(txt_dir='data/ehost_test_corpus/')
df = Vectorizer.docs_to_sents_df(docs, type_filter=set(), track_doc_name=True)
print(df)
assert (df.shape[0] == 12)
df = Vectorizer.docs_to_sents_df(docs, type_filter=set())
print(df)
df = Vectorizer.docs_to_sents_df(docs, sent_window=2)
assert (df.shape[0] == 20)
def __iter__(self, week=None):
with open(self.path + ".info") as info:
with open(self.path + ".title.bin") as title_bin:
for byte_string in Doc.read_bytes(title_bin):
info_line = info.readline()
comment_info = self._parse_info(info_line)
if not (week is None) and get_week(comment_info["timestamp"]) != week:
continue
if self.clean_deleted and comment_info["author"] == "[deleted]":
continue
if self.clean_bots and (is_bot(comment_info["author"]) or
comment_info["author"] in FILTERED_USERS):
continue
comment_info["doc"] = Doc(self._vocab).from_bytes(byte_string)
yield comment_info
def make_docs(nlp, batch, heads=True):
docs = []
for record in batch:
text = record["text"]
if "tokens" in record:
doc = Doc(nlp.vocab, words=record["tokens"])
else:
doc = nlp.make_doc(text)
if "heads" in record:
heads = record["heads"]
heads = numpy.asarray(heads, dtype="uint64")
heads = heads.reshape((len(doc), 1))
doc = doc.from_array([HEAD], heads)
if len(doc) >= 1 and len(doc) < 200:
docs.append(doc)
return docs
def get_tokens(model: Language, doc_id: int):
fn = os.path.join(settings.TOKEN_DIR, str(doc_id))
if not os.path.exists(fn):
raise ValueError(
"Document {doc_id} has not been preprocessed ({fn} does not exist)"
.format(**locals()))
return Doc(model.vocab).from_disk(fn)
def __iter__(self, week=None):
with open(self.path + ".bin", "rb") as bin:
with open(self.path + ".info") as info:
for byte_string in Doc.read_bytes(bin):
comment_info = self._parse_info(info.next())
if (not week is None) and get_week(comment_info["timestamp"]) != week:
continue
if self.clean_deleted and comment_info["author"] == "[deleted]":
continue
if self.clean_bots and (is_bot(comment_info["author"]) or
comment_info["author"] in FILTERED_USERS):
continue
doc = Doc(self._vocab).from_bytes(byte_string)
comment_info["doc"] = doc
comment_info["text"] = self._text_from_doc(doc)
yield comment_info
def decode(self, output: TaskOutput) -> TaskOutput:
# The dims are: batch, top_k, tags
output.tags: List[List[List[str]]] = [
self._decode_tags(paths) for paths in output.viterbi_paths
]
output.scores: List[List[float]] = [[score for tags, score in paths]
for paths in output.viterbi_paths]
output.entities: List[List[List[Dict]]] = []
output.tokens: List[List[Dict]] = []
# iterate over batch
for raw_text, k_tags in zip(output.raw_text, output.tags):
pre_tokenized = not isinstance(raw_text, str)
if pre_tokenized:
# compose spacy doc from tokens
doc = Doc(Vocab(), words=raw_text)
else:
doc = self.backbone.tokenizer.nlp(raw_text)
output.entities.append(
self._decode_entities(doc, k_tags, pre_tokenized))
output.tokens.append(
self._decode_tokens(doc) if not pre_tokenized else None)
if not any(output.tokens): # drop tokens field if no data
del output.tokens
del output.logits
del output.mask
del output.probs
del output.raw_text
del output.viterbi_paths
return output
def test_get_entity_via_match(en_vocab):
matcher = Matcher(en_vocab)
matcher.add_entity('TestEntity', attrs={u'Hello': u'World'})
assert matcher.n_patterns == 0
assert matcher(Doc(en_vocab, words=[u'Test', u'Entity'])) == []
matcher.add_pattern(u'TestEntity', [{ORTH: u'Test'}, {ORTH: u'Entity'}])
assert matcher.n_patterns == 1
matches = matcher(Doc(en_vocab, words=[u'Test', u'Entity']))
assert len(matches) == 1
assert len(matches[0]) == 4
ent_id, label, start, end = matches[0]
assert ent_id == matcher.vocab.strings[u'TestEntity']
assert label == 0
assert start == 0
assert end == 2
attrs = matcher.get_entity(ent_id)
assert attrs == {u'Hello': u'World'}
def test_parser_space_attachment_space(en_tokenizer, en_parser, text, length):
doc = Doc(en_parser.vocab, words=text)
assert len(doc) == length
with en_parser.step_through(doc) as _: # noqa: F841
pass
assert doc[0].is_space
for token in doc:
assert token.head.i == length - 1
def extract_entity(self, doc: Doc) -> List[Span]:
food_spans = []
for food in self.food_names:
food_index = doc.text.lower().find(food)
if food_index > -1:
food_spans.append(
doc.char_span(food_index, food_index + len(food)))
return food_spans
def __init__(self,
first_name_extension_name=FirstNameListMatcher.EXTENSION_NAME,
last_name_extension_name=LastNameListMatcher.EXTENSION_NAME):
self.token_extension_name = self.TOKEN_EXTENSION_NAME
self.span_extension_name = self.SPAN_EXTENSION_NAME
self.doc_extension_name = self.DOC_EXTENSION_NAME
self.first_name_extension_name = first_name_extension_name
self.last_name_extension_name = last_name_extension_name
if not Token.has_extension(self.token_extension_name):
Token.set_extension(self.token_extension_name,
default=self.ANOT_NONE)
if not Span.has_extension(self.span_extension_name):
Span.set_extension(self.span_extension_name,
getter=self.is_full_name_getter)
if not Doc.has_extension(self.doc_extension_name):
Doc.set_extension(self.doc_extension_name, default=[])
def test_match_zero_plus(matcher):
matcher.add('Quote', '', {}, [
[
{'ORTH': '"'},
{'OP': '*', 'IS_PUNCT': False},
{'ORTH': '"'}
]])
doc = Doc(matcher.vocab, 'He said , " some words " ...'.split())
assert len(matcher(doc)) == 1
def forward(texts, drop=0.):
if tokenized:
docs = [Doc(nlp.vocab, words) for words in texts]
else:
docs = [nlp(text) for text in texts]
features = [doc.to_array(attrs) for doc in docs]
def backward(d_features, sgd=None):
return d_features
return features, backward
def __call__(self, doc : Doc):
save_parsed = doc.is_parsed
doc.is_parsed = False
if self.split_matcher:
matches = self.split_matcher(doc)
for match_id, start, end in matches:
token = doc[end-1]
token.is_sent_start = True
if end-2>=0 and doc[end-2].is_sent_start is True:
doc[end-2].is_sent_start = False
if self.join_matcher:
matches = self.join_matcher(doc)
for match_id, start, end in matches:
# If there is a sent start in the match, just remove it
for token in doc[start:end]:
if token.is_sent_start:
token.is_sent_start = False
doc.is_parsed = save_parsed if doc.is_sentenced else True
return doc
def process_non_content_bearing_samples(
self, empty_samples: List[Tuple[int,
Text]]) -> List[Tuple[int, "Doc"]]:
"""Creates empty Doc-objects from zero-lengthed training samples strings."""
from spacy.tokens import Doc
n_docs = [(empty_sample[0], doc) for empty_sample, doc in zip(
empty_samples, [Doc(self.nlp.vocab) for doc in empty_samples])]
return n_docs
def test_override_sentiment(EN):
'''Test new span.sentiment property's default averaging behaviour'''
good = EN.vocab[u'good']
good.sentiment = 3.0
bad = EN.vocab[u'bad']
bad.sentiment = -2.0
doc = Doc(EN.vocab, [u'good', 'stuff', u'bad', u'stuff'])
doc.user_span_hooks['sentiment'] = lambda span: 10.0
good_stuff = doc[:2]
assert good_stuff.sentiment == 10.0
bad_stuff = doc[-2:]
assert bad_stuff.sentiment == 10.0
good_stuff_bad = doc[:-1]
assert good_stuff_bad.sentiment == 10.0
def test_read_bytes(nlp):
from spacy.tokens.doc import Doc
loc = '/tmp/test_serialize.bin'
with open(loc, 'wb') as file_:
file_.write(nlp(u'This is a document.').to_bytes())
file_.write(nlp(u'This is another.').to_bytes())
docs = []
with open(loc, 'rb') as file_:
for byte_string in Doc.read_bytes(file_):
docs.append(Doc(nlp.vocab).from_bytes(byte_string))
assert len(docs) == 2
def read_spacy_docs(spacy_vocab, filename):
"""
Stream ``spacy.Doc`` s from disk at ``filename`` where they were serialized
using Spacy's ``spacy.Doc.to_bytes()`` functionality.
Args:
spacy_vocab (``spacy.Vocab``): the spacy vocab object used to serialize
the docs in ``filename``
filename (str): /path/to/file on disk from which spacy docs will be streamed
Yields:
the next deserialized ``spacy.Doc``
"""
with io.open(filename, mode='rb') as f:
for bytes_string in SpacyDoc.read_bytes(f):
yield SpacyDoc(spacy_vocab).from_bytes(bytes_string)
dep_labels = []
while token.head is not token:
dep_labels.append(token.dep)
token = token.head
return dep_labels
for sentence in doc.sents:
for token in sentence:
print token
print token.orth
dep_labels = dependency_labels_to_root(token)
print dep_labels
for dep_label in dep_labels:
print nlp.vocab.strings[dep_label]
doc = nlp(u"Mr. Best flew to New York on Saturday morning.")
for ent in doc.ents:
print ent, ent.label_, ent.orth_
print ent.root, ent.root.head, ent.root.head.pos, nlp.vocab.strings[ent.root.head.pos], ent.root.head.lemma_
from spacy.tokens.doc import Doc
byte_string = doc.to_bytes()
open('moby_dick.bin', 'wb').write(byte_string)
doc = Doc(nlp.vocab)
for byte_string in Doc.read_bytes(open('moby_dick.bin', 'rb')):
doc.from_bytes(byte_string)
print doc