def normalization_props(self, path):
"""
Open Derived Normalization Properties
"""
DProps = {
'Full_Composition_Exclusion': ('full composition exclusion', True),
'Changes_When_NFKC_Casefolded':
('changes when NFKC casefolded', True),
'FC_NFKC': ('FC NFKC closure', None),
'NFKC_CF': ('NFKC casefold', None),
'NFD_QC': ('NFD quick check', 'N'),
'NFC_QC': ('NFC quick check', 'N'),
'NFKD_QC': ('NFKD quick check', 'N'),
'NFKC_QC': ('NFKC quick check', 'N'),
'Expands_On_NFD': ('expands on NFD', True),
'Expands_On_NFC': ('expands on NFC', True),
'Expands_On_NFKD': ('expands on NFKD', True),
'Expands_On_NFKC': ('expands on NFKC', True)
}
for L in open_scsv(path):
property = L[1]
assert property in DProps, "unknown normalization property %s" % property
if len(L) == 3:
ord_, property, mapping = L
yield DProps[property][0], ord_, mapping
else:
ord_, property = L
conv_property, mapping = DProps[property]
yield conv_property, ord_, mapping
def property_aliases(self, path):
"""
Open property Enum
Doesn't use a ord_, so can't be indexed
"""
DPropertyAliases = {}
for ord_, property in open_scsv('PropertyAliases.txt'):
DPropertyAliases[ord_] = property
def case_folding(self, path):
"""
Open Case Folding
"""
for ord_, status, mapping, __ignore__ in open_scsv(path):
# TODO: Should the properties be added with ord_/mapping reversed?
yield 'case folding status', ord_, status
yield 'case folding', ord_, mapping
def arabic_shaping(self, path):
"""
Open Arabic Shaping (StringData)
"""
for ord_, char_name, typ, group in open_scsv(path):
# character name is ignored here as it's the same as unicodedata.txt
yield 'arabic shaping type', ord_, typ
yield 'arabic shaping group', ord_, group
def property_value_aliases(self, path):
"""
Open property Value Enum
"""
DPropValAliases = {}
for L in open_scsv('PropertyValueAliases.txt'):
# FIXME: Provide support for L[3] where it exists?
alias, property, Value = L[0], L[1], L[2]
if not alias in DPropValAliases:
DPropValAliases[alias] = {}
DPropValAliases[alias][property] = Value
def normalization_corrections(self, path):
"""
Open Normalization Corrections
"""
for ord_, err_code_point, correct_code_point, unicode_ver in open_scsv(
path):
# TODO: Should the codepoints be corrected?
err_code_point = get_code_point(err_code_point)
correct_code_point = get_code_point(correct_code_point)
yield 'normalization corrections errors', ord_, err_code_point
yield 'normalization corrections corrected', ord_, correct_code_point
def special_casing(self, path):
"""
Open Special Casing
"""
for L in open_scsv(path):
ord_ = L[0]
def get_hex_encoding(i):
return [i[0] for i in get_L_encoding({}, i)] # HACK!
yield 'special casing lower', ord_, get_hex_encoding(L[1])
yield 'special casing title', ord_, get_hex_encoding(L[2])
yield 'special casing upper', ord_, get_hex_encoding(L[3])
if len(L) == 6:
# TODO: Add info to what this value means, e.g.
# "az Not_Before_Dot"
yield 'special casing condition list', ord_, L[4]
def simple(self, key, path):
"""
open a simple semicolon-separated file with two columns
"""
for ord_, value in open_scsv(path):
yield key, ord_, value
def composition_exclusions(self, path):
"""
Open Composition Exclusions (Boolean)
"""
for ord_, in open_scsv(path):
yield 'composition exclusions', ord_, True