def process():
for i, (sent, meta) in enumerate(
Conllu.load(corpus) if isinstance(corpus, str) else corpus):
tag_brat_len = len(TAG_BRAT)
for token in sent:
misc = token['MISC']
ne = None
ne_excess = set()
for feat, val in misc.items():
if feat.startswith(TAG_BRAT) and val == 'Yes':
if ne:
warnings.warn(
'Multiple brat entities in sent '
'{} (sent_id = {}), token {} ("{}"):'.format(
i, meta['sent_id'], token['ID'],
token['FORM']) +
': Entities {} and {}. Ignore the last one'.
format(ne, feat))
ne_excess.add(feat)
else:
ne = feat
if ne:
for ne_ in [ne] + list(ne_excess):
misc.pop(ne_)
misc[TAG_NE] = ne[tag_brat_len:]
yield sent, meta
def process():
for i, (sent, meta) in enumerate(
Conllu.load(corpus) if isinstance(corpus, str) else corpus):
for token in sent:
misc = token['MISC']
ne = None
ne_excess = set()
for feat, val in misc.items():
if feat.startswith(TAG):
if ne and ne != val:
warnings.warn(
'Multiple brat entities in sent '
'{} (sent_id = {}), token {} ("{}"):'.format(
i, meta['sent_id'], token['ID'],
token['FORM']) +
': Entities {} and {}. Ignore the last one'.
format(ne, val))
else:
ne = val
ne_excess.add(feat)
if ne:
if not keep_originals:
for ne_ in list(ne_excess):
misc.pop(ne_)
misc[TAG_NE] = ne
yield sent, meta
def process():
for sent, meta in Conllu.load(corpus) \
if isinstance(corpus, str) else \
corpus:
meta.pop('text', None)
sent_ = []
tags = []
for token in sent:
misc = token['MISC']
if token['FORM'] is None:
if TAGS_BRAT[0] in misc:
if TAGS_BRAT[0] not in tags:
tags.append(misc[TAGS_BRAT[0]])
elif TAGS_BRAT[1] in misc:
try:
tags.remove(misc[TAGS_BRAT[1]])
except:
pass
if sent_ and 'SpaceAfter' in misc:
sent_[-1]['MISC']['SpaceAfter'] = misc[
'SpaceAfter']
else:
sent_.append(token)
else:
for tag in tags:
misc[TAG_BRAT + tag] = 'Yes'
sent_.append(token)
yield sent_, meta
def load_conllu(*args, **kwargs):
"""Wrapper for ``Conllu.load()``"""
silent = kwargs.pop('silent', None)
if silent:
kwargs['log_file'] = None
elif 'log_file' not in kwargs:
kwargs['log_file'] = LOG_FILE
return Conllu.load(*args, **kwargs)
def split_corpus(corpus,
split=[.8, .1, .1],
save_split_to=None,
seed=None,
silent=False):
"""Split a *corpus* in the given proportion.
:param corpus: a name of file in CoNLL-U format or list/iterator of
sentences in Parsed CoNLL-U
:param split: list of sizes of the necessary *corpus* parts. If values
are of int type, they are interpreted as lengths of new
corpora in sentences; if values are float, they are
proportions of a given *corpus*. The types of the
*split* values can't be mixed: they are either all int,
or all float. The sum of float values must be less or
equals to 1; the sum of int values can't be greater than
the lentgh of the *corpus*
:param save_split_to: list of file names to save the result of the
*corpus* splitting. Can be `None` (default;
don't save parts to files) or its length must be
equal to the length of *split*
:param silent: if True, suppress output
:return: a list of new corpora
"""
assert save_split_to is None or len(save_split_to) == len(split), \
'ERROR: lengths of split and save_split_to must be equal'
isfloat = len([x for x in split if isinstance(x, float)]) > 0
if isfloat:
assert sum(split) <= 1, \
"ERROR: sum of split can't be greater that 1"
corpus = list(
Conllu.load(corpus, log_file=None if silent else LOG_FILE))
corpus_len = len(corpus)
if isfloat:
split = list(map(lambda x: round(corpus_len * x), split))
diff = corpus_len - sum(split)
if abs(diff) == 1:
split[-1] += diff
assert sum(split) <= corpus_len, \
"ERROR: sum of split can't be greater that corpus length"
random.seed(seed)
random.shuffle(corpus)
res = []
pos_b = 0
for i, sp in enumerate(split):
pos_e = pos_b + sp
corpus_ = corpus[pos_b:pos_e]
pos_b = pos_e
if save_split_to:
Conllu.save(corpus_, save_split_to[i])
res.append(corpus_)
return res
def get_conllu_fields(corpus=None,
fields=None,
word2idx=None,
unk_token=None,
with_empty=False,
silent=False):
"""Split corpus in CoNLL-U format to separate lists of tokens and tags.
:param corpus: the corpus in CoNLL-U or Parsed CoNLL-U format.
:param fields: list of CoNLL-U fields but 'FORM' to extract.
:type fields: list
:param word2idx: Word to Index dict. If not None, all words not from dict
will be skipped or replacet to *unk_token*
:type word2idx: dict({word: int})
:param unk_token: replacement for tokens that are not present in
*word2idx*.
:type unk_token: str
:param with_empty: don't skip empty sentences.
:param silent: suppress output.
:return: splitted corpus
:rtype: tuple(list([list([str|OrderedDict])]))
"""
if fields is None:
fields = []
if isinstance(corpus, str):
corpus = Conllu.load(corpus, **({'log_file': None} if silent else {}))
elif callable(corpus):
corpus = corpus()
sents = tuple([] for _ in range(len(fields) + 1))
for sent in corpus:
if isinstance(sent, tuple):
sent = sent[0]
for i, field in enumerate(
zip(*[(x['FORM'] if not word2idx or x['FORM'] in word2idx else
unk_token, *[
x[y[0]].get(y[1], y[2]) if len(y) >= 3 else x[y[0]].
get(y[1]) if len(y) == 2 else x[y[0]]
for y in [y.split(':') for y in fields]
]) for x in sent if x['FORM'] and '-' not in x['ID'] and
(not word2idx or x['FORM'] in word2idx or unk_token)])):
if field or with_empty:
sents[i].append(field)
return sents if fields else sents[0]
def embed_conllu_fields(corpus,
fields,
values,
empties=None,
nones=None,
silent=False):
if isinstance(corpus, str):
corpus = Conllu.load(corpus, **({'log_file': None} if silent else {}))
elif callable(corpus):
corpus = corpus()
if empties:
for i in empties:
values.insert(i, [])
if nones:
for i, j in nones:
values[i].insert(j, None)
for sentence, vals in zip(corpus, values):
sent = sentence[0] if isinstance(sentence, tuple) else sentence
for token, val in zip(sent, vals):
for field, val_ in [[fields, val]] \
if isinstance(fields, str) else \
zip(fields, val):
field = field.split(':')
if val_ is not None:
if len(field) >= 2:
if len(field) >= 3 and val_ == field[2]:
if field[1]:
token[field[0]].pop(field[1], None)
else:
token[field[0]] = None
else:
if field[1]:
token[field[0]][field[1]] = val_
else:
token[field[0]] = val_
else:
token[field[0]] = val_
yield sentence
#-*- encoding: utf-8 -*-
from copy import deepcopy
from corpuscula import Conllu
import glob
import os
from pathlib import Path
CONLL_DIR = r'C:\prj-git\_mine\ru_corner\_data\conll\newswire'
EDITED_DIR = '_0'
TOKEN = '%'
for fn in glob.glob(CONLL_DIR + '/*/*.txt', recursive=True):
print(fn)
corpus = list(Conllu.load(fn))
end_spaces = []
for sentence in corpus:
sent, meta = sentence
if 'par_text' in meta:
parts = meta['par_text'].split(TOKEN)
end_spaces = [x[-1:] == ' ' for x in parts[:-1]]
if not end_spaces:
continue
if 'text' not in meta:
continue
parts = meta['text'].split(TOKEN)
#!/usr/bin/python -u
#-*- encoding: utf-8 -*-
from copy import deepcopy
from corpuscula import Conllu
import glob
import os
from pathlib import Path
DIR = r'C:\prj-git\_mine\ru_corner\_data\conll\newswire'
TOKEN = '%'
log = open('_splitted', 'wt', encoding='utf-8')
parent_fn = None
for fn in glob.glob(DIR + '/*/*.txt', recursive=True):
corpus = list(Conllu.load(fn, fix=False, log_file=None))
path = Path(fn)
for idx, sentence in enumerate(corpus):
sent, meta = sentence
prev_id = None
for idx_, tok in enumerate(sent):
id_ = tok['ID']
if id_ == prev_id:
if parent_fn and parent_fn != fn:
print(file=log)
parent_fn = fn
print('{} ({}) - {} : {} / {} "{} {}"'
.format(meta['sent_id'], idx, id_,
path.parent.name, path.name,
def extract_conllu_fields(corpus,
fields=None,
word2idx=None,
unk_token=None,
with_empty=False,
return_nones=False,
silent=False):
"""Split corpus in CoNLL-U format to separate lists of tokens and tags.
:param corpus: the corpus in CoNLL-U or Parsed CoNLL-U format.
:param fields: list of CoNLL-U fields but 'FORM' to extract.
:type fields: list|str
:param word2idx: Word to Index dict. If not None, all words not from dict
will be skipped or replacet to *unk_token*
:type word2idx: dict({word: int})
:param unk_token: replacement for tokens that are not present in
*word2idx*.
:type unk_token: str
:param with_empty: don't skip empty sentences.
:param silent: suppress output.
:param return_nones: return indexes of filtered sentences and tokens
:return: splitted corpus
:rtype: tuple(list([list([str|OrderedDict])])), [ list([<empty sent idx]),
list([tuple(<empty token sent idx>, <empty token idx>)]) ]
"""
if fields is None:
fields = []
elif isinstance(fields, str):
fields = [fields]
if isinstance(corpus, str):
corpus = Conllu.load(corpus, **({'log_file': None} if silent else {}))
elif callable(corpus):
corpus = corpus()
sents = tuple([] for _ in range(len(fields) + 1))
empties, nones = [], []
for i, sent in enumerate(corpus):
if isinstance(sent, tuple):
sent = sent[0]
isempty = True
for j, field in enumerate(
zip(*[(x['FORM'] if not word2idx or x['FORM'] in word2idx else
unk_token, *[
x[y[0]].get(y[1], y[2]) if len(y) >= 3 and y[1] else
x[y[0]].get(y[1]) if len(y) ==
2 else x[y[0]] or y[2] if len(y) >= 3 else x[y[0]]
for y in [y.split(':') for y in fields]
]) for x in sent if x['FORM'] and '-' not in x['ID'] and
(not word2idx or x['FORM'] in word2idx or unk_token)])):
sents[j].append(field)
isempty = False
if isempty and return_nones:
empties.append(i)
if return_nones:
for j, x in enumerate(sent):
if not (x['FORM'] and '-' not in x['ID'] and
(not word2idx or x['FORM'] in word2idx or unk_token)):
nones.append((i, j))
return (*sents, *((empties, nones) if return_nones else [])) \
if fields or return_nones else \
sents[0]
def conllu_to_brat(corpus, txt_fn, ann_fn=None, spaces=3, short_spaces=1):
"""Converts *corpus* in CoNLL-U format to txt and ann files used by brat.
:param txt_fn: a path to the brat txt file.
:param ann_fn: a path to the brat ann file. If ``None`` (default), an
extension of *txt_fn* file will be changed to '.ann'.
:param save_to: a path where result will be stored. If ``None`` (default),
the function returns the result as a generator of Parsed
CoNLL-U data.
:param spaces: number of spaces to use as word delimiter.
:param short_spaces: number of spaces to use as word delimiter
inside multi-tokens (when ID field has a hyphen
inside).
Note, that we create empty `.ann` files. Use this function to get initial
data for annotation."""
fn, fe = os.path.splitext(txt_fn)
if fe != '.txt':
print('WARNING: Extension of txt_fn must be ".txt"', file=sys.stderr)
if ann_fn is None:
ann_fn = fn + '.ann'
_, fe = os.path.splitext(ann_fn)
if fe != '.ann':
print('WARNING: Extension of ann_fn must be ".ann"', file=sys.stderr)
with io.open(txt_fn, 'wt', encoding='utf-8', newline='\n') as out_f, \
open(ann_fn, 'w'):
for sent_no, sent in enumerate(
Conllu.load(corpus, fix=False, log_file=None)):
if sent_no:
print(file=out_f)
if 'newpar id' in sent[1]:
print(file=out_f)
is_next = None
short_start = short_end = None
for tok in sent[0]:
id_, form, misc = tok['ID'], tok['FORM'], tok['MISC']
if '.' in id_:
continue
if '-' in id_:
short_start, short_end, *_ = id_.split('-')
continue
if is_next:
print(' ' * (short_spaces
if short_end and not short_start else spaces),
end='',
file=out_f)
has_entity = False
for feat, value in misc.items():
if feat.startswith('Entity'):
assert not has_entity
# workaround because brat can't display emojies
# correctly
form = '[emo]' if feat == 'EntityEmoji' else value
has_entity = True
is_next = True
if short_end:
short_start = None
if id_ == short_end:
short_end = None
print(form, end='', file=out_f)
def process():
def unmask(text):
return text.replace(r'\{}'.format(BRAT_TEXT_BOUND_START_MARK[-1]),
BRAT_TEXT_BOUND_START_MARK[-1]) \
.replace(r'\{}'.format(SEP1), SEP1) \
.replace('__', ' ').replace(r'\_', '_')
for sent, meta in Conllu.load(corpus) \
if isinstance(corpus, str) else \
corpus:
meta.pop('text', None)
if 'par_text' in meta:
meta['par_text'] = RE_BRAT.sub('', meta['par_text'])
sent_ = []
anns = []
for token in sent:
misc = token['MISC']
if token['FORM'] is None:
if BRAT_START_TAG in misc:
assert BRAT_START_TAG not in anns
assert misc[BRAT_START_TAG][0] == 'T', \
'ERROR: Invalid annotation type "{}"' \
.format(misc[BRAT_START_TAG])
anns.append(misc[BRAT_START_TAG])
elif BRAT_END_TAG in misc:
anns_ = []
for ann in anns:
prefix = misc[BRAT_END_TAG] + SEP2
anns = list(
filter(lambda x: not x.startswith(prefix),
anns))
try:
tags.remove(misc[BRAT_END_TAG])
except:
pass
if sent_ and 'SpaceAfter' in misc:
sent_[-1]['MISC']['SpaceAfter'] = \
misc['SpaceAfter']
else:
sent_.append(token)
else:
for ann in anns:
ann = ann.split(SEP1 + SEP1)
entity, ann_ = ann[0], ann[1:]
tid, name = entity.split(SEP2)
assert tid.startswith('T'), \
'ERROR: Unrecognized annotation {}'.format(ann)
misc[BRAT_TAG + tid] = name
for ann in ann_:
if ann.startswith('R'):
ann_id, name, role = ann.split(SEP2)
misc[BRAT_TAG + ann_id] = \
tid + SEP3 + name + SEP3 + role
elif ann.startswith('*'):
ann_id, name = ann.split(SEP2)
misc[BRAT_TAG + ann_id] = tid + SEP3 + name
elif ann.startswith('E'):
ann_id, name, role = ann.split(SEP2)
val = tid + SEP3 + name
if role:
val += SEP3 + role
misc[BRAT_TAG + ann_id] = val
elif ann.startswith('A'):
ann_id, name, value = ann.split(SEP2)
val = tid + SEP3 + name
if value:
val += SEP3 + value
misc[BRAT_TAG + ann_id] = val
elif ann.startswith('N'):
ann_id, service_name, service_id, title = \
ann.split(SEP2, maxsplit=3)
misc[BRAT_TAG + ann_id] = \
tid + SEP3 + service_name \
+ SEP3 + service_id + SEP3 + unmask(title)
elif ann.startswith('#'):
ann_id, note = ann.split(SEP2, maxsplit=1)
misc[BRAT_TAG + ann_id] = \
tid + SEP3 + unmask(note)
else:
raise ValueError('ERROR: Unknown annotation '
'type')
#misc[BRAT_TAG + ann] = 'Yes'
sent_.append(token)
yield sent_, meta