class TestSymmetry(unittest.TestCase):
def setUp(self):
self.lgr = LGR()
# Configure log system to redirect validation logs to local attribute
self.log_output = StringIO()
ch = logging.StreamHandler(self.log_output)
ch.setLevel(logging.DEBUG)
logging.getLogger('lgr.validate').addHandler(ch)
def test_empty_lgr(self):
success, result = check_symmetry(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertEqual(len(log_content), 0)
self.assertTrue(success)
self.assertDictEqual(result, {'description': 'Testing symmetry',
'repertoire': []})
def test_no_symmetric_in_repertoire(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0062])
success, result = check_symmetry(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertGreater(len(log_content), 0)
self.assertEqual(log_content,
"CP U+0061: Variant U+0062 is not in repertoire.\n")
self.assertFalse(success)
self.assertDictEqual(result, {'description': 'Testing symmetry',
'repertoire': [{'char': self.lgr.get_char([0x0061]),
'variant': self.lgr.get_variant([0x0061], (0x0062, ))[0],
'type': 'not-in-repertoire'}]})
def test_no_symmetric_in_variants(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0062])
self.lgr.add_cp([0x0062])
success, result = check_symmetry(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertGreater(len(log_content), 0)
self.assertEqual(log_content,
'CP U+0062 should have CP U+0061 in its variants.\n')
self.assertFalse(success)
self.assertDictEqual(result, {'description': 'Testing symmetry',
'repertoire': [{'char': self.lgr.get_variant([0x0061], (0x0062, ))[0],
'variant': self.lgr.get_char([0x0061]),
'type': 'missing'}]})
def test_symmetry_ok(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0062])
self.lgr.add_cp([0x0062])
self.lgr.add_variant([0x0062], [0x0061])
success, result = check_symmetry(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertEqual(len(log_content), 0)
self.assertTrue(success)
self.assertDictEqual(result, {'description': 'Testing symmetry',
'repertoire': []})
class TestTransitivity(unittest.TestCase):
def setUp(self):
self.lgr = LGR()
# Configure log system to redirect validation logs to local attribute
self.log_output = StringIO()
ch = logging.StreamHandler(self.log_output)
ch.setLevel(logging.DEBUG)
logging.getLogger('lgr.validate').addHandler(ch)
def test_empty_lgr(self):
success, result = check_transitivity(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertEqual(len(log_content), 0)
self.assertTrue(success)
self.assertDictEqual(result, {'description': 'Testing transitivity',
'repertoire': []})
def test_no_variants(self):
self.lgr.add_cp([0x0061])
self.lgr.add_cp([0x0062])
success, result = check_transitivity(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertEqual(len(log_content), 0)
self.assertTrue(success)
self.assertDictEqual(result, {'description': 'Testing transitivity',
'repertoire': []})
def test_no_transitivity(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0062])
self.lgr.add_cp([0x0062])
self.lgr.add_variant([0x0062], [0x0063])
self.lgr.add_cp([0x0063])
success, result = check_transitivity(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertGreater(len(log_content), 0)
self.assertEqual(log_content,
'CP U+0061 should have CP U+0063 in its variants.\n')
self.assertFalse(success)
self.assertDictEqual(result, {'description': 'Testing transitivity',
'repertoire': [{'char': self.lgr.get_char([0x0061]),
'variant': self.lgr.get_variant([0x0062], (0x0063, ))[0]}]})
def test_transitivity_ok(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0062])
self.lgr.add_variant([0x0061], [0x0063])
self.lgr.add_cp([0x0062])
self.lgr.add_variant([0x0062], [0x0063])
self.lgr.add_cp([0x0063])
success, result = check_transitivity(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertEqual(len(log_content), 0)
self.assertTrue(success)
self.assertDictEqual(result, {'description': 'Testing transitivity',
'repertoire': []})
def test_merge_chars(self):
merged_lgr = LGR()
# Need to merge references first - OK since tested in previous test
reference_mapping = {}
merge_references(self.lgr_1, 'fr', merged_lgr, reference_mapping)
merge_references(self.lgr_2, 'und-Khmer', merged_lgr, reference_mapping)
merge_chars(self.lgr_1, 'fr', merged_lgr, reference_mapping, [])
# Simple variant changed to blocked
cp = merged_lgr.get_char(0x0041)
self.assertIn('1', cp.references)
variants = list(cp.get_variants())
self.assertEqual(len(variants), 1)
var = variants[0]
self.assertEqual(var.cp, (0x0061, )),
self.assertEqual(var.type, 'blocked')
# Complete merge
merge_chars(self.lgr_2, 'und-Khmer', merged_lgr, reference_mapping, [])
self._test_merged_chars(merged_lgr)
class TestPopulate(unittest.TestCase):
def setUp(self):
self.lgr = LGR()
# Configure log system to redirect validation logs to local attribute
self.log_output = StringIO()
ch = logging.StreamHandler(self.log_output)
ch.setLevel(logging.INFO)
logger = logging.getLogger('lgr.populate')
logger.addHandler(ch)
logger.setLevel(logging.INFO)
def test_no_symmetric_in_repertoire(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0062])
populate_lgr(self.lgr)
log_content = self.log_output.getvalue()
self.assertEqual(
"Add missing code point 'U+0062' in LGR as it is a variant of 'U+0061'\n"
"Add code point 'U+0061' as variant of 'U+0062' for symmetry\n",
log_content)
self.assertIn(0x0062, self.lgr.repertoire)
new_variant = self.lgr.get_char([0x0062])
self.assertEqual([(0x0061, )],
[c.cp for c in new_variant.get_variants()])
def test_no_symmetric_in_repertoire_twice(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0062])
self.lgr.add_variant([0x0061], [0x0063])
self.lgr.add_cp([0x0062])
self.lgr.add_variant([0x0062], [0x0061])
self.lgr.add_variant([0x0062], [0x0063])
populate_lgr(self.lgr)
log_content = self.log_output.getvalue()
self.assertEqual(
"Add missing code point 'U+0063' in LGR as it is a variant of 'U+0061'\n"
"Add code point 'U+0061' as variant of 'U+0063' for symmetry\n"
"Add code point 'U+0062' as variant of 'U+0063' for symmetry\n",
log_content)
self.assertIn(0x0063, self.lgr.repertoire)
new_variant = self.lgr.get_char([0x0063])
self.assertEqual([(0x0061, ), (0x0062, )],
[c.cp for c in new_variant.get_variants()])
def test_no_symmetric_in_variants(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0062])
self.lgr.add_cp([0x0062])
populate_lgr(self.lgr)
log_content = self.log_output.getvalue()
self.assertEqual(
"Add code point 'U+0061' as variant of 'U+0062' for symmetry\n",
log_content)
cp = self.lgr.get_char([0x0062])
self.assertEqual([(0x0061, )], [c.cp for c in cp.get_variants()])
def test_no_transitivity(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0062])
self.lgr.add_cp([0x0062])
self.lgr.add_variant([0x0062], [0x0063])
self.lgr.add_cp([0x0063])
populate_lgr(self.lgr)
log_content = self.log_output.getvalue()
self.assertEqual(
"Add code point 'U+0061' as variant of 'U+0062' for symmetry\n"
"Add code point 'U+0062' as variant of 'U+0063' for symmetry\n"
"Add code point 'U+0063' as variant of 'U+0061' for transitivity with 'U+0062'\n"
"Add code point 'U+0061' as variant of 'U+0063' for transitivity with 'U+0062'\n",
log_content)
cp = self.lgr.get_char([0x0061])
self.assertEqual([(0x0062, ), (0x0063, )],
[c.cp for c in cp.get_variants()])
cp = self.lgr.get_char([0x0062])
self.assertEqual([(0x0061, ), (0x0063, )],
[c.cp for c in cp.get_variants()])
cp = self.lgr.get_char([0x0063])
self.assertEqual([(0x0061, ), (0x0062, )],
[c.cp for c in cp.get_variants()])
class TestLGRCore(unittest.TestCase):
def setUp(self):
unidb = IDNADatabase('6.3.0')
self.lgr = LGR(unicode_database=unidb)
def test_add_single_cp_list(self):
self.lgr.add_cp([0x0061])
self.assertIn(0x0061, self.lgr.repertoire)
def test_add_single_cp_int(self):
self.lgr.add_cp(0x0061)
self.assertIn(0x0061, self.lgr.repertoire)
def test_add_cp_sequence(self):
self.lgr.add_cp([0x0061, 0x0062])
self.assertIn([0x0061, 0x0062], self.lgr.repertoire)
self.assertNotIn(0x0061, self.lgr.repertoire)
self.assertNotIn(0x0062, self.lgr.repertoire)
def test_add_multiple_cp_sequences(self):
self.lgr.add_cp([0x0061, 0x0062])
self.lgr.add_cp([0x0061, 0x0062, 0x0063])
self.assertIn([0x0061, 0x0062], self.lgr.repertoire)
self.assertIn([0x0061, 0x0062, 0x0063], self.lgr.repertoire)
self.assertNotIn(0x0061, self.lgr.repertoire)
self.assertNotIn(0x0062, self.lgr.repertoire)
self.assertNotIn(0x0063, self.lgr.repertoire)
def test_add_cp_in_repertoire(self):
self.lgr.add_cp([0x0061])
self.assertRaises(CharAlreadyExists, self.lgr.add_cp, [0x0061])
self.assertRaises(CharAlreadyExists, self.lgr.add_cp, 0x0061)
def test_add_cp_validation(self):
validation_lgr = LGR()
validation_lgr.add_cp([0x0061])
self.lgr.add_cp([0x0061],
validating_repertoire=validation_lgr,
override_repertoire=False)
self.assertRaises(NotInRepertoire,
self.lgr.add_cp, [0x0062],
validating_repertoire=validation_lgr,
override_repertoire=False)
def test_add_cp_validation_override(self):
validation_lgr = LGR()
validation_lgr.add_cp([0x0061])
self.lgr.add_cp([0x0061],
validating_repertoire=validation_lgr,
override_repertoire=False)
self.lgr.add_cp([0x0062],
validating_repertoire=validation_lgr,
override_repertoire=True)
self.assertIn(0x0062, self.lgr.repertoire)
def test_del_single_cp_list(self):
self.lgr.add_cp(0x0061)
self.lgr.del_cp([0x0061])
self.assertNotIn(0x0061, self.lgr.repertoire)
def test_del_single_cp_int(self):
self.lgr.add_cp([0x0061])
self.lgr.del_cp(0x0061)
self.assertNotIn(0x0061, self.lgr.repertoire)
def test_del_cp_sequence(self):
self.lgr.add_cp([0x0061, 0x0062])
self.lgr.del_cp([0x0061, 0x0062])
self.assertEqual(len(self.lgr.repertoire), 0)
def test_del_cp_sequence_with_cp(self):
self.lgr.add_cp([0x0061, 0x0062])
self.assertRaises(NotInLGR, self.lgr.del_cp, 0x0061)
self.assertRaises(NotInLGR, self.lgr.del_cp, 0x0062)
self.assertIn([0x0061, 0x0062], self.lgr.repertoire)
def test_add_cp_when_not_when(self):
self.lgr.add_cp([0x0061], when='w1')
with self.assertRaises(CharInvalidContextRule) as cm:
self.lgr.add_cp([0x0062], when='w2', not_when='nw1')
the_exception = cm.exception
self.assertEqual(the_exception.cp, [0x0062])
self.lgr.add_cp([0x0062], not_when='nw2')
with self.assertRaises(CharInvalidContextRule) as cm:
self.lgr.add_cp([0x0063], when='w3', not_when='nw3')
the_exception = cm.exception
self.assertEqual(the_exception.cp, [0x0063])
def test_add_range(self):
self.lgr.add_range(0x0061, 0x007A)
for cp in range(0x0061, 0x007A + 1):
self.assertIn(cp, self.lgr.repertoire)
def test_add_range_in_repertoire(self):
self.lgr.add_range(0x0061, 0x007A)
self.assertRaises(CharAlreadyExists, self.lgr.add_range, 0x0061,
0x007A)
def test_add_range_validation(self):
validation_lgr = LGR()
for cp in range(0x0061, 0x007A + 1):
validation_lgr.add_cp(cp)
self.lgr.add_range(0x0061,
0x007A,
validating_repertoire=validation_lgr,
override_repertoire=False)
self.assertRaises(NotInRepertoire,
self.lgr.add_range,
0x00F8,
0x00FF,
validating_repertoire=validation_lgr,
override_repertoire=False)
def test_add_range_validation_with_range(self):
validation_lgr = LGR()
validation_lgr.add_range(0x0061, 0x007A)
self.lgr.add_range(0x0061,
0x007A,
validating_repertoire=validation_lgr,
override_repertoire=False)
self.assertRaises(NotInRepertoire,
self.lgr.add_range,
0x00F8,
0x00FF,
validating_repertoire=validation_lgr,
override_repertoire=False)
def test_add_range_validation_override(self):
validation_lgr = LGR()
for cp in range(0x0061, 0x007A):
validation_lgr.add_cp(cp)
self.lgr.add_range(0x0031,
0x0032,
validating_repertoire=validation_lgr,
override_repertoire=True)
self.assertIn(0x0031, self.lgr.repertoire)
def test_add_range_when_not_when(self):
self.lgr.add_range(0x0061, 0x0065, when='w1')
with self.assertRaises(RangeInvalidContextRule) as cm:
self.lgr.add_range(0x0066, 0x007A, when='w2', not_when='nw1')
the_exception = cm.exception
self.assertEqual(the_exception.first_cp, 0x0066)
self.assertEqual(the_exception.last_cp, 0x007A)
self.lgr.add_range(0x0066, 0x007A, not_when='nw2')
with self.assertRaises(RangeInvalidContextRule) as cm:
self.lgr.add_range(0x01BD, 0x01C3, when='w3', not_when='nw3')
the_exception = cm.exception
self.assertEqual(the_exception.first_cp, 0x01BD)
self.assertEqual(the_exception.last_cp, 0x01C3)
def test_expand_ranges(self):
self.lgr.add_range(0x0061, 0x007A)
for cp in range(0x0061, 0x007A + 1):
self.assertIsInstance(self.lgr.get_char(cp), RangeChar)
self.lgr.add_range(0x01BD, 0x01C3)
for cp in range(0x01BD, 0x01C3 + 1):
self.assertIsInstance(self.lgr.get_char(cp), RangeChar)
self.lgr.expand_ranges()
for cp in range(0x0061, 0x007A + 1):
char = self.lgr.get_char(cp)
self.assertIsInstance(char, Char)
self.assertNotIsInstance(char, RangeChar)
for cp in range(0x01BD, 0x01C3 + 1):
char = self.lgr.get_char(cp)
self.assertIsInstance(char, Char)
self.assertNotIsInstance(char, RangeChar)
def test_expand_range(self):
self.lgr.add_range(0x0061, 0x007A)
for cp in range(0x0061, 0x007A + 1):
self.assertIsInstance(self.lgr.get_char(cp), RangeChar)
self.lgr.expand_range(0x0061, 0x007A)
for cp in range(0x0061, 0x007A + 1):
char = self.lgr.get_char(cp)
self.assertIsInstance(char, Char)
self.assertNotIsInstance(char, RangeChar)
def test_add_variant_in_repertoire(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0030])
self.assertRaises(VariantAlreadyExists, self.lgr.add_variant, [0x0061],
[0x0030])
def test_add_variant_validation(self):
validation_lgr = LGR()
validation_lgr.add_cp([0x0061])
validation_lgr.add_cp([0x0030])
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0030])
self.assertRaises(NotInRepertoire,
self.lgr.add_variant, [0x0061], [0x0062],
validating_repertoire=validation_lgr,
override_repertoire=False)
def test_add_variant_when_not_when(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0030], when='w1')
with self.assertRaises(VariantInvalidContextRule) as cm:
self.lgr.add_variant([0x0061], [0x0031], when='w2', not_when='nw1')
the_exception = cm.exception
self.assertEqual(the_exception.cp, [0x0061])
self.assertEqual(the_exception.variant, [0x0031])
self.lgr.add_variant([0x0061], [0x0030], not_when='nw2')
with self.assertRaises(VariantInvalidContextRule) as cm:
self.lgr.add_variant([0x0061], [0x0031], when='w3', not_when='nw3')
the_exception = cm.exception
self.assertEqual(the_exception.cp, [0x0061])
self.assertEqual(the_exception.variant, [0x0031])
def test_del_cp_validation_override(self):
validation_lgr = LGR()
validation_lgr.add_cp([0x0061])
validation_lgr.add_cp([0x0030])
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0030])
self.lgr.add_variant([0x0061], [0x0062],
validating_repertoire=validation_lgr,
override_repertoire=True)
self.assertIn((0x0062, ), self.lgr.repertoire[0x0061]._variants)
def test_get_variants(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0030])
variants = self.lgr.get_variants([0x0061])
self.assertIsInstance(variants, types.GeneratorType)
variant_list = list(variants)
self.assertEqual(len(variant_list), 1)
def test_check_range_no_modification(self):
self.lgr.check_range(0x0060, 0x007F)
self.assertEqual(len(self.lgr.repertoire), 0)
def test_check_range(self):
self.lgr.add_cp([0x0061])
self.lgr.add_cp([0x007A])
codepoints = self.lgr.check_range(0x0060, 0x007F)
for result in codepoints:
cp = result[0]
prop = result[1]
if cp == 0x060 or cp >= 0x007B:
self.assertIsInstance(prop, CharInvalidIdnaProperty)
elif cp == 0x0061 or cp == 0x007A:
self.assertIsInstance(prop, CharAlreadyExists)
else:
self.assertIsNone(prop)
def test_add_codepoints(self):
self.lgr.add_codepoints([c for c in range(0x0061, 0x007A + 1)] +
[0x0107] + [0x0137, 0x0138])
expected_output = [
RangeChar(0x061, 0x0061, 0x007A),
Char(0x0107),
RangeChar(0x0137, 0x0137, 0x0138)
]
self.assertEqual(expected_output, list(self.lgr.repertoire))
def test_tags_on_codepoint(self):
self.lgr.add_cp([0x0061], tag=['t1', 't2'])
with self.assertRaises(LGRFormatException) as cm:
self.lgr.add_cp([0x0062], tag=['t1', 't1'])
the_exception = cm.exception
self.assertEqual(the_exception.reason,
LGRFormatException.LGRFormatReason.DUPLICATE_TAG)
def test_tags_on_codepoint_sequence(self):
with self.assertRaises(LGRFormatException) as cm:
self.lgr.add_cp([0x0061, 0x0062], tag=['t1'])
the_exception = cm.exception
self.assertEqual(the_exception.reason,
LGRFormatException.LGRFormatReason.SEQUENCE_NO_TAG)
def test_tags_on_range(self):
self.lgr.add_range(0x0061, 0x0062, tag=['t1', 't2'])
with self.assertRaises(LGRFormatException) as cm:
self.lgr.add_range(0x0063, 0x0064, tag=['t1', 't1'])
the_exception = cm.exception
self.assertEqual(the_exception.reason,
LGRFormatException.LGRFormatReason.DUPLICATE_TAG)
def test_list_types(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0030], variant_type='BLOCK')
self.lgr.add_variant([0x0061], [0x0031], variant_type='VALID')
self.lgr.add_variant([0x0061], [0x0032], variant_type='BLOCK')
self.assertEquals(self.lgr.types, set(['BLOCK', 'VALID']))
def test_del_reference(self):
ref_id_1 = self.lgr.add_reference("Test - 1")
ref_id_2 = self.lgr.add_reference("Test - 2")
self.lgr.add_cp([0x0061], ref=[ref_id_1])
self.lgr.add_cp([0x0062], ref=[ref_id_1, ref_id_2])
self.lgr.del_reference(ref_id_1)
self.assertNotIn(ref_id_1, self.lgr.reference_manager)
self.assertEquals(self.lgr.get_char([0x0061]).references, [])
self.assertEquals(self.lgr.get_char([0x0062]).references, [ref_id_2])
def test_add_cp_duplicate_reference(self):
ref_id = self.lgr.add_reference("Test - 1")
with self.assertRaises(DuplicateReference) as cm:
self.lgr.add_cp([0x0061], ref=[ref_id, ref_id])
the_exception = cm.exception
self.assertEqual(the_exception.cp, [0x0061])
def test_add_range_duplicate_reference(self):
ref_id = self.lgr.add_reference("Test - 1")
with self.assertRaises(DuplicateReference) as cm:
self.lgr.add_range(0x0061, 0x0062, ref=[ref_id, ref_id])
the_exception = cm.exception
self.assertEqual(the_exception.cp, 0x0061)
def test_add_variant_duplicate_reference(self):
self.lgr.add_cp([0x0061])
ref_id = self.lgr.add_reference("Test - 1")
with self.assertRaises(DuplicateReference) as cm:
self.lgr.add_variant([0x0061], [0x0062], ref=[ref_id, ref_id])
the_exception = cm.exception
self.assertEqual(the_exception.cp, [0x0061])
def test_generate_variants(self):
self.lgr.add_cp([0x0061])
self.lgr.add_cp([0x0062])
self.lgr.add_cp([0x0063])
self.lgr.add_cp([0x0064])
self.lgr.add_variant([0x0061], [0x0070], variant_type="type0")
self.lgr.add_variant([0x0062], [0x0071], variant_type="type1")
self.lgr.add_variant([0x0062], [0x0072], variant_type="type2")
self.assertEqual([], list(self.lgr._generate_label_variants([])))
self.assertEqual([], list(self.lgr._generate_label_variants([0x0063])))
self.assertEqual(
[], list(self.lgr._generate_label_variants([0x0063, 0x0064])))
self.assertEqual(
set([((0x0071, 0x0063), frozenset(['type1']), False),
((0x0072, 0x0063), frozenset(['type2']), False)]),
set(self.lgr._generate_label_variants([0x0062, 0x0063])))
self.assertEqual(
set([
((0x0061, 0x0062), frozenset(), False),
((0x0061, 0x0071), frozenset(['type1']), False),
((0x0061, 0x0072), frozenset(['type2']), False),
((0x0070, 0x0062), frozenset(['type0']), False),
((0x0070, 0x0071), frozenset(['type0', 'type1']), True),
((0x0070, 0x0072), frozenset(['type0', 'type2']), True),
]), set(self.lgr._generate_label_variants([0x0061, 0x0062])))
self.assertEqual(
set([
((0x0061, 0x0062, 0x0062), frozenset(), False),
((0x0061, 0x0062, 0x0071), frozenset(['type1']), False),
((0x0061, 0x0062, 0x0072), frozenset(['type2']), False),
((0x0061, 0x0071, 0x0062), frozenset(['type1']), False),
((0x0061, 0x0071, 0x0071), frozenset(['type1']), False),
((0x0061, 0x0071, 0x0072), frozenset(['type1',
'type2']), False),
((0x0061, 0x0072, 0x0062), frozenset(['type2']), False),
((0x0061, 0x0072, 0x0071), frozenset(['type1',
'type2']), False),
((0x0061, 0x0072, 0x0072), frozenset(['type2']), False),
((0x0070, 0x0062, 0x0062), frozenset(['type0']), False),
((0x0070, 0x0062, 0x0071), frozenset(['type0',
'type1']), False),
((0x0070, 0x0062, 0x0072), frozenset(['type0',
'type2']), False),
((0x0070, 0x0071, 0x0062), frozenset(['type0',
'type1']), False),
((0x0070, 0x0071, 0x0071), frozenset(['type0',
'type1']), True),
((0x0070, 0x0071, 0x0072),
frozenset(['type0', 'type1', 'type2']), True),
((0x0070, 0x0072, 0x0062), frozenset(['type0',
'type2']), False),
((0x0070, 0x0072, 0x0071),
frozenset(['type0', 'type1', 'type2']), True),
((0x0070, 0x0072, 0x0072), frozenset(['type0',
'type2']), True),
]), set(self.lgr._generate_label_variants([0x0061, 0x0062,
0x0062])))
def test_generate_variants_reflexive(self):
self.lgr.add_cp([0x0061])
self.lgr.add_cp([0x0062])
self.lgr.add_cp([0x0063])
self.lgr.add_variant([0x0062], [0x0062], variant_type="reflexive")
self.lgr.add_variant([0x0063], [0x0070], variant_type="type")
self.assertEqual([], list(self.lgr._generate_label_variants([])))
self.assertEqual([], list(self.lgr._generate_label_variants([0x0061])))
self.assertEqual([((0x0062, ), frozenset(['reflexive']), True)],
list(self.lgr._generate_label_variants([0x0062])))
self.assertEqual(
set([
((0x0062, 0x0063), frozenset(['reflexive']), False),
((0x0062, 0x0070), frozenset(['reflexive', 'type']), True),
]), set(self.lgr._generate_label_variants([0x0062, 0x0063])))
def test_label_simple(self):
self.lgr.add_cp([0x0061])
self.lgr.add_cp([0x0062, 0x0063])
self.lgr.add_range(0x0064, 0x0068)
valid_labels = ([0x0061], [0x0062, 0x0063], [0x0064], [0x0068],
[0x0061, 0x0064], [0x0061, 0x0062, 0x0063,
0x0064], [0x0062, 0x0063, 0x0068])
invalid_labels = (([0x0060], [], [0x0060]), ([0x0069], [], [0x0069]),
([0x0062], [], [0x0062]), ([0x0063], [], [0x0063]),
([0x0061, 0x0062], [0x0061], [0x0062]))
for label in valid_labels:
self.assertEqual((True, label, []),
self.lgr._test_preliminary_eligibility(label))
for (label, label_part, not_in_lgr) in invalid_labels:
self.assertEqual((False, label_part, not_in_lgr),
self.lgr._test_preliminary_eligibility(label))
def test_label_eligibility_multiple_choices(self):
self.lgr.add_cp([0x0061])
self.lgr.add_cp([0x0061, 0x0062, 0x0063])
self.lgr.add_cp([0x0064])
self.assertEqual(self.lgr._test_preliminary_eligibility([0x0062]),
(False, [], [0x0062]))
self.assertEqual(
self.lgr._test_preliminary_eligibility(
[0x0061, 0x0062, 0x0063, 0x0064]),
(True, [0x0061, 0x0062, 0x0063, 0x0064], []))
def test_label_delayed_eligibilty(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0061], 'block')
self.lgr.add_cp([0x0062])
self.lgr.add_variant([0x0062], [0x0062], 'invalid')
self.lgr.add_cp([0x0063, 0x0064])
self.lgr.add_variant([0x0063, 0x0064], [0x0063, 0x0064], 'invalid')
self.assertEqual(self.lgr._test_label_disposition([0x0062]),
('invalid', 0))
self.assertEqual(self.lgr._test_label_disposition([0x0063, 0x0064]),
('invalid', 0))
self.assertEqual(self.lgr._test_label_disposition([0x0061, 0x0062]),
('invalid', 0))
def test_label_length(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0061], 'disp')
self.lgr.add_cp([0x0062])
self.lgr.add_variant([0x0062], [0x0062], 'disp')
self.assertEqual(PROTOCOL_LABEL_MAX_LENGTH,
self.lgr.max_label_length())
for i in range(80):
self.lgr.add_variant([0x0062], [0x074D + i], 'disp')
# 41: mean number of variants per character
self.assertEqual(int(math.log(MAX_NUMBER_GENERATED_VARIANTS, 41)),
self.lgr.max_label_length())
class TestRebuildLGR(unittest.TestCase):
DEFAULT_UNICODE_VERSION = '6.3.0'
def setUp(self):
self.lgr = LGR()
def test_empty_lgr(self):
__, result = rebuild_lgr(self.lgr, {})
self.assertDictEqual(result, {'description': 'Rebuilding LGR with Unicode version {}'.format(
self.DEFAULT_UNICODE_VERSION),
'repertoire': {}})
def test_lgr_non_default_unicode(self):
self.lgr.metadata.set_unicode_version('6.2.0')
__, result = rebuild_lgr(self.lgr, {})
self.assertDictEqual(result, {'description': 'Rebuilding LGR with Unicode version 6.2.0',
'repertoire': {}})
def test_lgr_validating_repertoire(self):
validating_repertoire = LGR(name='validating')
__, result = rebuild_lgr(self.lgr, {'validating_repertoire': validating_repertoire})
self.assertDictEqual(result, {'description': "Rebuilding LGR with Unicode version {} "
"and validating repertoire '{}'".format(
self.DEFAULT_UNICODE_VERSION, validating_repertoire),
'repertoire': {}})
def test_lgr_unidb_same_unicode(self):
unidb = IDNADatabase('6.3.0')
__, result = rebuild_lgr(self.lgr, {'unidb': unidb})
self.assertDictEqual(result, {'description': 'Rebuilding LGR with Unicode version {}'.format(
self.DEFAULT_UNICODE_VERSION),
'repertoire': {}})
def test_lgr_unidb_different_unicode(self):
unidb = IDNADatabase('6.2.0')
__, result = rebuild_lgr(self.lgr, {'unidb': unidb})
self.assertDictEqual(result, {'description': 'Rebuilding LGR with Unicode version {}'.format(
self.DEFAULT_UNICODE_VERSION),
'generic': "Target Unicode version {} differs from UnicodeDatabase {}".format(
self.DEFAULT_UNICODE_VERSION,
'6.2.0'),
'repertoire': {}})
def test_lgr_wrong_range_char(self):
self.lgr.add_range(0x0060, 0x0063, force=True)
r = RangeChar(0x0060, 0x0060, 0x0063)
unidb = IDNADatabase(self.DEFAULT_UNICODE_VERSION)
self.lgr.unicode_database = unidb
_, result = rebuild_lgr(self.lgr, {'unidb': unidb})
errors = result.get('repertoire', {}).get(r, {'errors': []})['errors']
self.assertEqual(len(errors), 1)
self.assertIsInstance(errors[0], CharInvalidIdnaProperty)
self.assertDictEqual(result, {'description': 'Rebuilding LGR with Unicode version {}'.format(
self.DEFAULT_UNICODE_VERSION),
'repertoire': {r: {'errors': errors}}})
def test_lgr_wrong_char(self):
self.lgr.add_cp(0x0060)
char = self.lgr.get_char([0x0060])
unidb = IDNADatabase(self.DEFAULT_UNICODE_VERSION)
self.lgr.unicode_database = unidb
_, result = rebuild_lgr(self.lgr, {'unidb': unidb})
errors = result.get('repertoire', {}).get(char, {'errors': []})['errors']
self.assertEqual(len(errors), 1)
self.assertIsInstance(errors[0], CharInvalidIdnaProperty)
self.assertDictEqual(result, {'description': 'Rebuilding LGR with Unicode version {}'.format(
self.DEFAULT_UNICODE_VERSION),
'repertoire': {char: {'errors': errors}}})
def test_lgr_wrong_variant(self):
self.lgr.add_cp(0x0061)
self.lgr.add_variant(0x0061, 0x0060)
char = self.lgr.get_char([0x0061])
var = char.get_variant((0x0060, ))[0]
unidb = IDNADatabase(self.DEFAULT_UNICODE_VERSION)
self.lgr.unicode_database = unidb
_, result = rebuild_lgr(self.lgr, {'unidb': unidb})
errors = result.get('repertoire', {}).get(char, {}).get('variants', {}).get(var, [])
self.assertEqual(len(errors), 1)
self.assertIsInstance(errors[0], CharInvalidIdnaProperty)
self.assertDictEqual(result, {'description': 'Rebuilding LGR with Unicode version {}'.format(
self.DEFAULT_UNICODE_VERSION),
'repertoire': {char: {'variants': {var: errors}}}})
def test_lgr_ok(self):
self.lgr.add_range(0x0061, 0x0063, force=True)
self.lgr.add_cp(0x0064)
self.lgr.add_cp(0x0065)
self.lgr.add_variant(0x0064, 0x0065)
self.lgr.add_variant(0x0065, 0x0064)
unidb = IDNADatabase(self.DEFAULT_UNICODE_VERSION)
self.lgr.unicode_database = unidb
_, result = rebuild_lgr(self.lgr, {'unidb': unidb})
self.assertDictEqual(result, {'description': 'Rebuilding LGR with Unicode version {}'.format(
self.DEFAULT_UNICODE_VERSION),
'repertoire': {}})
class TestConditionalVariants(unittest.TestCase):
def setUp(self):
self.lgr = LGR()
# Configure log system to redirect validation logs to local attribute
self.log_output = StringIO()
ch = logging.StreamHandler(self.log_output)
ch.setLevel(logging.DEBUG)
logging.getLogger('lgr.validate').addHandler(ch)
def test_empty_lgr(self):
success, result = check_conditional_variants(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertEqual(len(log_content), 0)
self.assertTrue(success)
self.assertDictEqual(result, {'description': 'Testing conditional variants',
'repertoire': []})
def test_no_variants(self):
self.lgr.add_cp([0x0061])
self.lgr.add_cp([0x0062])
success, result = check_conditional_variants(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertEqual(len(log_content), 0)
self.assertTrue(success)
self.assertDictEqual(result, {'description': 'Testing conditional variants',
'repertoire': []})
def test_no_rule_when(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0062], when="when-rule")
success, result = check_conditional_variants(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertGreater(len(log_content), 0)
self.assertEqual(log_content,
"CP U+0061: Variant 'U+0062' \"when\" attribute "
"'when-rule' is not an existing rule name.\n")
self.assertFalse(success)
var = self.lgr.get_variant([0x0061], (0x0062, ))[0]
self.assertDictEqual(result, {'description': 'Testing conditional variants',
'repertoire': [{'char': self.lgr.get_char([0x0061]),
'variant': var,
'rule_type': 'when',
'rule': var.when}]})
def test_no_rule_not_when(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0062], not_when="not-when-rule")
success, result = check_conditional_variants(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertGreater(len(log_content), 0)
self.assertEqual(log_content,
"CP U+0061: Variant 'U+0062' \"not-when\" attribute "
"'not-when-rule' is not an existing rule name.\n")
self.assertFalse(success)
var = self.lgr.get_variant([0x0061], (0x0062, ))[0]
self.assertDictEqual(result, {'description': 'Testing conditional variants',
'repertoire': [{'char': self.lgr.get_char([0x0061]),
'variant': var,
'rule_type': 'not-when',
'rule': var.not_when}]})
def test_no_rule_when_not_when(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0062], when="when-rule", not_when="not-when-rule", force=True)
success, result = check_conditional_variants(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertGreater(len(log_content), 0)
self.assertEqual(log_content,
"CP U+0061: Variant 'U+0062' \"when\" attribute "
"'when-rule' is not an existing rule name.\n"
"CP U+0061: Variant 'U+0062' \"not-when\" attribute "
"'not-when-rule' is not an existing rule name.\n")
self.assertFalse(success)
var = self.lgr.get_variant([0x0061], (0x0062, ))[0]
self.assertDictEqual(result, {'description': 'Testing conditional variants',
'repertoire': [{'char': self.lgr.get_char([0x0061]),
'variant': var,
'rule_type': 'when',
'rule': var.when},
{'char': self.lgr.get_char([0x0061]),
'variant': var,
'rule_type': 'not-when',
'rule': var.not_when}
]})
def test_conditional_when_ok(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0062], when="when-rule")
self.lgr.rules.append("when-rule")
success, result = check_conditional_variants(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertEqual(len(log_content), 0)
self.assertTrue(success)
self.assertDictEqual(result, {'description': 'Testing conditional variants',
'repertoire': []})
def test_conditional_not_when_ok(self):
self.lgr.add_cp([0x0061])
self.lgr.add_variant([0x0061], [0x0062], not_when="not-when-rule")
self.lgr.rules.append("not-when-rule")
success, result = check_conditional_variants(self.lgr, {})
log_content = self.log_output.getvalue()
self.assertEqual(len(log_content), 0)
self.assertTrue(success)
self.assertDictEqual(result, {'description': 'Testing conditional variants',
'repertoire': []})