def test_collation_order_for_partitioning(self):
'Testing collation ordering for partitioning'
for group in [
(self.ae, ('Smith', 'Šaa', 'Solženicyn', 'Štepánek')),
(self.ane, ('11', '011')),
(self.ae, ('102 Smith', '100 Smith')),
(self.ane, ('100 Smith', '0100 Smith')),
]:
last = None
assert_func = group[0]
for x in group[1]:
order, _ = icu.non_numeric_sort_collator().collation_order(x)
if last is not None:
assert_func(
last, order,
f'Order for {x} not correct: {last} != {order}')
last = order
self.ae(
dict(
icu.partition_by_first_letter(
['A1', '', 'a1', '\U0001f431', '\U0001f431x'])), {
' ': [''],
'A': ['A1', 'a1'],
'\U0001f431': ['\U0001f431', '\U0001f431x']
})
def test_collation_order(self):
'Testing collation ordering'
for group in [
(self.ae, ('Šaa', 'Smith', 'Solženicyn', 'Štepánek')),
(self.ae, ('11', '011')),
(self.ane, ('2', '1')),
(self.ae, ('100 Smith', '0100 Smith')),
]:
last = None
assert_func = group[0]
for x in group[1]:
order, _ = icu.numeric_collator().collation_order(x)
if last is not None:
assert_func(
last, order, 'Order for %s not correct: %s != %s' %
(x, last, order))
last = order
self.ae(
dict(
icu.partition_by_first_letter(
['A1', '', 'a1', '\U0001f431', '\U0001f431x'])), {
' ': [''],
'A': ['A1', 'a1'],
'\U0001f431': ['\U0001f431', '\U0001f431x']
})
def test_collation_order(self):
"Testing collation ordering"
for group in [("Šaa", "Smith", "Solženicyn", "Štepánek"), ("01", "1"), ("1", "11", "13")]:
last = None
for x in group:
order, length = icu.numeric_collator().collation_order(x)
if last is not None:
self.ae(last, order)
last = order
self.ae(
dict(icu.partition_by_first_letter(["A1", "", "a1", "\U0001f431", "\U0001f431x"])),
{" ": [""], "A": ["A1", "a1"], "\U0001f431": ["\U0001f431", "\U0001f431x"]},
)
def test_collation_order(self):
'Testing collation ordering'
for group in [
('Šaa', 'Smith', 'Solženicyn', 'Štepánek'),
('01', '1'),
]:
last = None
for x in group:
order, length = icu.numeric_collator().collation_order(x)
if last is not None:
self.ae(last, order, 'Order for %s not correct: %s != %s' % (x, last, order))
last = order
self.ae(dict(icu.partition_by_first_letter(['A1', '', 'a1', '\U0001f431', '\U0001f431x'])),
{' ':[''], 'A':['A1', 'a1'], '\U0001f431':['\U0001f431', '\U0001f431x']})
def test_collation_order(self):
'Testing collation ordering'
for group in [
('Šaa', 'Smith', 'Solženicyn', 'Štepánek'),
('01', '1'),
]:
last = None
for x in group:
order, length = icu.numeric_collator().collation_order(x)
if last is not None:
self.ae(last, order, 'Order for %s not correct: %s != %s' % (x, last, order))
last = order
self.ae(dict(icu.partition_by_first_letter(['A1', '', 'a1', '\U0001f431', '\U0001f431x'])),
{' ':[''], 'A':['A1', 'a1'], '\U0001f431':['\U0001f431', '\U0001f431x']})
def process_index(field, index, xe_fields, log, XPath, expand):
'''
We remove all the word generated index markup and replace it with our own
that is more suitable for an ebook.
'''
styles = []
heading_text = index.get('heading', None)
heading_style = 'IndexHeading'
start_pos = None
for elem in field.contents:
if elem.tag.endswith('}p'):
s = XPath('descendant::pStyle/@w:val')(elem)
if s:
styles.append(s[0])
p = elem.getparent()
if start_pos is None:
start_pos = (p, p.index(elem))
p.remove(elem)
xe_fields = get_applicable_xe_fields(index, xe_fields, XPath, expand)
if not xe_fields:
return [], []
if heading_text is not None:
groups = partition_by_first_letter(xe_fields, key=itemgetter('text'))
items = []
for key, fields in groups.iteritems():
items.append(key), items.extend(fields)
if styles:
heading_style = styles[0]
else:
items = sorted(xe_fields, key=lambda x: sort_key(x['text']))
hyperlinks = []
blocks = []
for item in reversed(items):
is_heading = not isinstance(item, dict)
style = heading_style if is_heading else None
p, t = make_block(expand, style, *start_pos)
if is_heading:
text = heading_text
if text.lower().startswith('a'):
text = item + text[1:]
t.text = text
else:
hyperlinks.append(add_xe(item, t, expand))
blocks.append(p)
return hyperlinks, blocks
def process_index(field, index, xe_fields, log, XPath, expand):
'''
We remove all the word generated index markup and replace it with our own
that is more suitable for an ebook.
'''
styles = []
heading_text = index.get('heading', None)
heading_style = 'IndexHeading'
start_pos = None
for elem in field.contents:
if elem.tag.endswith('}p'):
s = XPath('descendant::pStyle/@w:val')(elem)
if s:
styles.append(s[0])
p = elem.getparent()
if start_pos is None:
start_pos = (p, p.index(elem))
p.remove(elem)
xe_fields = get_applicable_xe_fields(index, xe_fields, XPath, expand)
if not xe_fields:
return [], []
if heading_text is not None:
groups = partition_by_first_letter(xe_fields, key=itemgetter('text'))
items = []
for key, fields in groups.iteritems():
items.append(key), items.extend(fields)
if styles:
heading_style = styles[0]
else:
items = sorted(xe_fields, key=lambda x:sort_key(x['text']))
hyperlinks = []
blocks = []
for item in reversed(items):
is_heading = not isinstance(item, dict)
style = heading_style if is_heading else None
p, t = make_block(expand, style, *start_pos)
if is_heading:
text = heading_text
if text.lower().startswith('a'):
text = item + text[1:]
t.text = text
else:
hyperlinks.append(add_xe(item, t, expand))
blocks.append(p)
return hyperlinks, blocks
