def merge_items(*items):
alignment = ','.join(i.alignment for i in items if i.alignment)
content = ','.join(i.content for i in items if i.content)
segmentation = ','.join(i.segmentation for i in items if i.segmentation)
if segmentation and (alignment or content):
raise ValueError(
'Cannot merge items defining segmentation and another '
'reference attribute.')
base = items[0]
base.text = ' '.join(i.text for i in items)
base.attributes['line'] = ' '.join(i.attributes['line'] for i in items)
if alignment: base.alignment = alignment
if content: base.content = content
if segmentation: base.segmentation = segmentation
pri_tags = set()
sec_tags = set()
for item in items:
tags = get_tags(item)
if tags[0]:
pri_tags.add(tags[0])
sec_tags.update(tags[1:])
tag = '-'.join(sorted(pri_tags)).replace('G-L', 'L-G')
if sec_tags:
tag = '{}+{}'.format(tag, '+'.join(sorted(sec_tags)))
base.attributes['tag'] = tag
def remove_example_numbers(items):
# IGT-initial numbers (e.g. '1.' '(1)', '5a.', '(ii)')
def removable(m):
start, end = m.span()
end -= 1 # ignore the required final space
mtext = m.group('exnum')
for item in items:
tags = get_tags(item)
if tags[0] not in ('L', 'G', 'T', 'L-G', 'G-T', 'L-T', 'L-G-T'):
continue
text = (item.text or '')[start:end]
if text != mtext and text.strip() != '':
return False
return True
for item in items:
tags = get_tags(item)
if tags[0] in ('L-G', 'L-T', 'G-T', 'L-G-T'):
item.text = ex_num_re.sub(whitespace, item.text)
elif tags[0] in ('L', 'G', 'T'):
m = ex_num_re.match(item.text)
while m and removable(m):
item.text = ex_num_re.sub(whitespace, item.text)
m = ex_num_re.match(item.text)
return items
def normalize_items(base_tier, norm_id):
# first make copies of the original items
items = copy_items(base_tier.items)
items = remove_blank_items(items) # don't bother with blank lines
items = rejoin_continuations(items)
items = rejoin_translations(items)
items = remove_citations(items)
items = remove_language_name(items, base_tier.igt)
items = remove_example_numbers(items)
items = remove_blank_items(items) # in case previous created blanks
for item in items:
# and set the copy's alignments to their current ID (changed later)
item.alignment = item.id
rejoin_hyphenated_grams(item)
extract_judgment(item)
# any remaining tag=B items should be changed to tag=M
# (because they aren't blank)
tags = get_tags(item)
if tags[0] == 'B':
tags = ['M'] + tags[1:]
item.attributes['tag'] = '+'.join(tags)
items = separate_secondary_translations(items)
items = dewrap_lines(items)
items = unquote_translations(items)
items = shift_left(items, tags=('L', 'G', 'L-G', 'L-T', 'G-T', 'L-G-T'))
for i, item in enumerate(items):
item.id = '{}{}'.format(norm_id, i + 1)
return items
def extract_judgment(item):
tags = get_tags(item)
if tags[0] == 'M' or 'CR' in tags:
return
match = re.match(r'^\s*([*?#]+)[^/]+$', item.text, re.U)
if match:
item.attributes['judgment'] = match.group(1)
item.text = re.sub(r'^(\s*)[*?#]+\s*', r'\1', item.text, re.U)
def merge_lines(items):
"""
Return the lines with corrupted and split lines merged.
Merge corrupted lines if:
* Both lines have the +CR tag
* Both lines have one other tag in common
* The lines are sequential
* tokens in one line align to whitespace in the other
TODO:
* Can we allow some non-whitespace overlap, in which case the
token would be inserted in the closest match
* Can we recombine diacritics with the letter it came from?
E.g. is this usually accents or umlauts on vowels?
* Is there anything that can be done about intraline corruption?
E.g. when spaces are inserted w i t h i n words
"""
n = len(items)
# nothing to do if there's just 1 line
if n < 2:
return items
newitems = [items[0]]
for i in range(1, n):
# lines are pairs of attributes and content
prev = newitems[-1]
cur = items[i]
p_tags = get_tags(prev)
c_tags = get_tags(cur)
# if no non-CR tags are shared
if 'CR' not in c_tags or \
len(set(p_tags).intersection(c_tags).difference(['CR'])) == 0:
newitems.append(cur)
continue
merged = bit_merge(prev.text or '', cur.text or '')
if merged is not None:
# there's no OrderedSet, but OrderedDict will do
tags = OrderedDict((t, 1) for t in p_tags + c_tags)
del tags['CR'] # assume we fixed the problem?
line_nums = ' '.join(
[prev.attributes.get('line'),
cur.attributes.get('line')])
prev.attributes['tag'] = '+'.join(tags)
prev.attributes['line'] = line_nums
prev.text = merged
return newitems
def merge_lines(items):
"""
Return the lines with corrupted and split lines merged.
Merge corrupted lines if:
* Both lines have the +CR tag
* Both lines have one other tag in common
* The lines are sequential
* tokens in one line align to whitespace in the other
TODO:
* Can we allow some non-whitespace overlap, in which case the
token would be inserted in the closest match
* Can we recombine diacritics with the letter it came from?
E.g. is this usually accents or umlauts on vowels?
* Is there anything that can be done about intraline corruption?
E.g. when spaces are inserted w i t h i n words
"""
n = len(items)
# nothing to do if there's just 1 line
if n < 2:
return items
newitems = [items[0]]
for i in range(1,n):
# lines are pairs of attributes and content
prev = newitems[-1]
cur = items[i]
p_tags = get_tags(prev)
c_tags = get_tags(cur)
# if no non-CR tags are shared
if 'CR' not in c_tags or \
len(set(p_tags).intersection(c_tags).difference(['CR'])) == 0:
newitems.append(cur)
continue
merged = bit_merge(prev.text or '', cur.text or '')
if merged is not None:
# there's no OrderedSet, but OrderedDict will do
tags = OrderedDict((t,1) for t in p_tags + c_tags)
del tags['CR'] # assume we fixed the problem?
line_nums = ' '.join([prev.attributes.get('line'),
cur.attributes.get('line')])
prev.attributes['tag'] = '+'.join(tags)
prev.attributes['line'] = line_nums
prev.text = merged
return newitems
def unquote_translations(items):
for item in items:
tags = get_tags(item)
if tags[0] == 'T':
item.text = re.sub(r'^\s*[{}]?'.format(OPENQUOTES), '', item.text,
re.U)
item.text = re.sub(r'[{}]\s*$'.format(CLOSEQUOTES), '', item.text,
re.U)
return items
def rejoin_continuations(items):
new_items = []
for item in items:
tags = get_tags(item)
if tags[0] == 'C' and new_items:
item.text = item.text.lstrip()
item.attributes['tag'] = item.attributes['tag'][1:] # remove C
merge_items(new_items[-1], item)
else:
new_items.append(item)
return new_items
def dewrap_lines(items):
# look for patterns like L G L G and join them to L G
# then look for T T and join them to T if they don't look like alternates
unwrapped = []
used = set()
sig = []
for item in items:
tags = get_tags(item)
if tags[0] in ('L', 'G', 'T'):
sig.append(item.attributes['tag'])
sig = ' '.join(sig)
if (any(x in sig for x in ('L G L G ', 'L G T L G T', 'G G ', 'L L '))
and not any(x in sig for x in ('L+', 'L-', 'G+', 'G-'))):
# likely patterns for wrapping without other noise
ls = [item for item in items if item.attributes.get('tag') == 'L']
gs = [item for item in items if item.attributes.get('tag') == 'G']
for l_, g_ in zip_longest(ls, gs):
if l_ is not None and g_ is not None:
maxlen = max([len(l_.text), len(g_.text)])
l_.text = l_.text.ljust(maxlen)
g_.text = g_.text.ljust(maxlen)
if ls:
merge_items(*ls)
unwrapped.append(ls[0])
if gs:
merge_items(*gs)
unwrapped.append(gs[0])
used.update(id(x) for x in ls + gs)
# add everything unused up to the first translation
for item in items:
if item.attributes.get('tag') in ('T', 'T+AC'):
break
elif id(item) not in used:
unwrapped.append(item)
used.add(id(item))
# now do translations
if (any(x in sig for x in ('L G T L G T', 'T T+AC', 'T T+LN', 'T T'))
and not any(x in sig for x in ('+EX', '+LT', '+AL', 'T+CR'))):
# translations that appear wrapped and not alternates
ts = [
item for item in items
if item.attributes.get('tag') in ('T', 'T+AC', 'T+LN')
]
if ts:
merge_items(*ts)
unwrapped.append(ts[0])
used.update(id(x) for x in ts)
# finally add anything unused
for item in items:
if id(item) not in used:
unwrapped.append(item)
used.add(id(item))
return unwrapped
def removable(m):
start, end = m.span()
end -= 1 # ignore the required final space
mtext = m.group('exnum')
for item in items:
tags = get_tags(item)
if tags[0] not in ('L', 'G', 'T', 'L-G', 'G-T', 'L-T', 'L-G-T'):
continue
text = (item.text or '')[start:end]
if text != mtext and text.strip() != '':
return False
return True
def remove_language_name(items, igt):
new_items = []
lgcode = xp.find(igt, LANG_CODE_PATH)
lgname = xp.find(igt, LANG_NAME_PATH)
lgtoks = []
if lgcode and '?' not in lgcode and '*' not in lgcode:
codes = set(lgcode.split(':')) # split up complex codes
codes.update(map(str.upper, list(codes)))
codes.update(map(str.lower, list(codes)))
lgtoks.extend(codes)
if lgname and '?' not in lgname:
lgtoks.append(lgname)
lgtoks.append(lgname.upper())
if re.search('[- ]', lgname, re.U): # abbreviation for multiword names
lgtoks.append(''.join(ln[0]
for ln in re.split(r'[- ]+', lgname, re.U)))
if re.search(r'^\w{3}', lgname, re.U):
lgtoks.append(lgname[:3])
if lgtoks:
sig = '|'.join(re.escape(t) for t in lgtoks)
start_lg_re = re.compile(r'^\s*[(\[]?({})[)\]]?'.format(sig), re.U)
end_lg_re = re.compile(r'[(\[]?({})[)\]]?\s*$'.format(sig), re.U)
for item in items:
new_items.append(item) # add now; might be modified later
tags = get_tags(item)
if tags[0] != 'M':
orig = item.text
m = start_lg_re.match(item.text)
if m:
meta_item = Item(id=item.id,
text=m.group(0).strip(),
attributes=dict(item.attributes))
meta_item.attributes['tag'] = 'M+LN'
new_items.append(meta_item)
item.text = start_lg_re.sub(whitespace, item.text)
m = end_lg_re.search(item.text)
if m:
meta_item = Item(id=item.id,
text=m.group(0).strip(),
attributes=dict(item.attributes))
meta_item.attributes['tag'] = 'M+LN'
items.append(meta_item)
item.text = end_lg_re.sub(whitespace, item.text).rstrip()
if 'LN' in tags and item.text != orig:
tags.remove('LN')
item.attributes['tag'] = '+'.join(tags)
else:
new_items = items
return new_items
def remove_citations(items):
def removable(m, t, i):
# citation matches are removable if they don't look like
# translation alternates or bracketed glosses
if t in ('L', 'G'):
start, end = m.span()
other = None
if t == 'L': # look down then up for nearest G
others = items[i + 1:] + items[i - 1::-1]
t2 = 'G'
else: # look up then down for nearest L
others = items[i - 1:] + items[i - 1::-1]
t2 = 'L'
other = next((i for i in others if get_tags(i)[0] == t2), None)
if other and (other.text or '')[start:end].strip() != '':
return False
elif re.match(r'\s*[{}].*[{}]\s*$'.format(OPENQUOTES, CLOSEQUOTES),
m.group('inner1') or m.group('inner2'), re.U):
return False
return True
new_items = []
for i, item in enumerate(items):
new_items.append(item) # add now; text might be modified later
tags = get_tags(item)
if tags[0] not in ('L', 'G', 'T', 'L-G', 'L-T', 'L-G-T'):
continue
match = citation_re.search(item.text)
if (match and removable(match, tags[0], i)):
meta_item = Item(id=item.id,
text=match.group(0).strip(),
attributes=item.attributes)
m_tags = ['M']
item.text = citation_re.sub('', item.text).rstrip()
if 'AC' in tags:
tags.remove('AC')
m_tags.append('AC')
elif 'LN' in tags:
tags.remove('LN')
m_tags.append('LN')
elif 'CN' in tags:
tags.remove('CN')
m_tags.append('CN')
# what about other tags? LN, CN, EX
item.attributes['tag'] = '+'.join(tags)
meta_item.attributes['tag'] = '+'.join(m_tags)
new_items.append(meta_item)
return new_items
def rejoin_hyphenated_grams(item):
# there may be morphemes separated by hyphens, but with intervening
# spaces; slide the token over (e.g. "dog- NOM" => "dog-NOM ")
tags = get_tags(item)
delims = {'L': '-=', 'L-G': '-=', 'G': '-=.'}
if tags[0] in delims:
pattern = r'(\S*(?:\s*[{}]\s*\S*)+)'.format(delims[tags[0]])
text = item.text
toks = []
pos = 0
for match in list(re.finditer(pattern, text, re.U)):
start, end = match.span()
toks.append(text[pos:start])
toks.append(text[start:end].replace(' ', ''))
pos = end
toks.append(text[pos:len(text)])
item.text = ''.join(toks).rstrip()
def removable(m, t, i):
# citation matches are removable if they don't look like
# translation alternates or bracketed glosses
if t in ('L', 'G'):
start, end = m.span()
other = None
if t == 'L': # look down then up for nearest G
others = items[i + 1:] + items[i - 1::-1]
t2 = 'G'
else: # look up then down for nearest L
others = items[i - 1:] + items[i - 1::-1]
t2 = 'L'
other = next((i for i in others if get_tags(i)[0] == t2), None)
if other and (other.text or '')[start:end].strip() != '':
return False
elif re.match(r'\s*[{}].*[{}]\s*$'.format(OPENQUOTES, CLOSEQUOTES),
m.group('inner1') or m.group('inner2'), re.U):
return False
return True
def rejoin_translations(items):
# rejoin translation lines if they don't start with some kind of
# speaker indicator, quote, or other
new_items = []
prev_is_t = False
prev_end = False
for item in items:
tags = get_tags(item)
is_t = tags[0] == 'T' and 'DB' not in tags and 'CR' not in tags
marked = re.match(r'^\s*[(\[]?\s*\S+\s*\.?\s*[)\]]?\s*:', item.text,
re.U)
if prev_is_t and is_t and not marked and not prev_end:
item.text = item.text.lstrip()
merge_items(new_items[-1], item)
else:
new_items.append(item)
prev_is_t = is_t
end_match = re.search(r'[{}] *\)* *$'.format(CLOSEQUOTES), item.text)
prev_end = end_match is not None
return new_items
def separate_secondary_translations(items):
# sometimes translation lines with secondary translations are marked
# as +DB even if they are for the same, single IGT
for item in items:
tags = get_tags(item)
if tags[0] in ('L', 'G', 'L-G') and 'DB' in tags[1:]:
# don't attempt
return items
indent = min_indent(items, tags=('L', 'G', 'L-G', 'L-G-T', 'G-T'))
new_items = []
for item in items:
tags = get_tags(item)
text = item.text
if (tags[0] == 'T' and 'CR' not in tags[1:]):
text = re.sub(
r'([{cq}])\s*(\s|/)\s*([{oq}])'.format(oq=OPENQUOTES,
cq=CLOSEQUOTES),
r'\1 \2 \3', text, re.I | re.U)
matches = [
m for m in basic_quoted_trans_re.finditer(text)
if m.group('t').strip()
]
sub_items = []
if matches:
pos = 0
bare_T_seen = False
last_i = len(matches) - 1
for i, match in enumerate(matches):
start, end = match.start(), match.end()
t = match.group('t')
if i == last_i and re.search(r'\w|\d', text[end:], re.U):
t += text[match.end():]
pre = text[pos:match.start()]
# some instances have bad matches... try to avoid with
# a hard limit of 30 chars for the note or note is 2x
# size of t
prelen = len(pre.strip())
if prelen > 30 or prelen >= (2 * len(t.strip())):
sub_items = []
new_items.append(item)
break
new_tags = list(tags)
if re.search(r'lit(?:eral(?:ly)?)?', pre):
if 'LT' not in new_tags: new_tags.append('LT')
elif (re.search(r'(or|also|ii+|\b[bcd]\.)[ :,]', pre)
or bare_T_seen):
if 'AL' not in new_tags: new_tags.append('AL')
else:
bare_T_seen = True
attrs = dict(item.attributes)
if match.group('judg'):
attrs['judgment'] = match.group('judg')
if re.search(r'\w|\d', pre, re.U):
attrs['note'] = pre.strip()
attrs['tag'] = '+'.join(new_tags)
sub_items.append(
Item(id=item.id + '_{}'.format(i + 1),
attributes=attrs,
text=t))
pos = end
new_items.extend(sub_items)
else:
new_items.append(item)
else:
new_items.append(item)
return new_items