def __init__(self, *a, **kw):
super(Locale, self).__init__(*a, **kw)
self.currency_formats['amount_only'] = MONEY_AMOUNT_FORMAT
delta_p = self.currency_formats['standard'].pattern
minus_sign = self.number_symbols.get('minusSign', '-')
plus_sign = self.number_symbols.get('plusSign', '+')
if ';' in delta_p:
pos, neg = delta_p.split(';')
assert len(neg) > len(pos)
assert minus_sign in neg
pos = neg.replace(minus_sign, plus_sign)
self.currency_delta_pattern = parse_pattern('%s;%s' % (pos, neg))
else:
self.currency_delta_pattern = parse_pattern('{0}{2};{1}{2}'.format(
plus_sign, minus_sign, delta_p))
def test_parse_pattern_negative():
# No negative format specified
np = numbers.parse_pattern(u'¤#,##0.00')
assert np.prefix == (u'¤', u'-¤')
assert np.suffix == (u'', u'')
# Negative format is specified
np = numbers.parse_pattern(u'¤#,##0.00;(¤#,##0.00)')
assert np.prefix == (u'¤', u'(¤')
assert np.suffix == (u'', u')')
# Negative sign is a suffix
np = numbers.parse_pattern(u'¤ #,##0.00;¤ #,##0.00-')
assert np.prefix == (u'¤ ', u'¤ ')
assert np.suffix == (u'', u'-')
def format_scientific_field(spec, prec, number, locale):
prec = SCIENTIFIC_DECIMAL_DIGITS if prec is None else int(prec)
format_ = u'0.%sE+000' % (u'#' * prec)
pattern = parse_pattern(format_)
decimal_symbol = get_decimal_symbol(locale)
string = pattern.apply(number, locale).replace(u'.', decimal_symbol)
return string.lower() if spec.islower() else string
def format_scientific_field(spec, prec, number, locale):
prec = SCIENTIFIC_DECIMAL_DIGITS if prec is None else int(prec)
format_ = u'0.%sE+000' % (u'#' * prec)
pattern = parse_pattern(format_)
decimal_symbol = get_decimal_symbol(locale)
string = pattern.apply(number, locale).replace(u'.', decimal_symbol)
return string.lower() if spec.islower() else string
def load_i18n(project_root, tell_sentry):
# Load the locales
localeDir = os.path.join(project_root, 'i18n', 'core')
locales = LOCALES
for file in os.listdir(localeDir):
try:
parts = file.split(".")
if not (len(parts) == 2 and parts[1] == "po"):
continue
lang = parts[0]
with open(os.path.join(localeDir, file)) as f:
l = locales[lang.lower()] = Locale(lang)
c = l.catalog = read_po(f)
c.plural_func = get_function_from_rule(c.plural_expr)
try:
l.countries = make_sorted_dict(COUNTRIES, l.territories)
except KeyError:
l.countries = COUNTRIES
try:
l.languages_2 = make_sorted_dict(LANGUAGES_2, l.languages)
except KeyError:
l.languages_2 = LANGUAGES_2
except Exception as e:
tell_sentry(e, {})
# Add aliases
for k, v in list(locales.items()):
locales.setdefault(ALIASES.get(k, k), v)
locales.setdefault(ALIASES_R.get(k, k), v)
for k, v in list(locales.items()):
locales.setdefault(k.split('_', 1)[0], v)
# Patch the locales to look less formal
locales['fr'].currency_formats[None] = parse_pattern('#,##0.00\u202f\xa4')
locales['fr'].currency_symbols['USD'] = '$'
def parse_currency_formats(data, tree):
currency_formats = data.setdefault('currency_formats', {})
for currency_format in tree.findall('.//currencyFormats'):
if _should_skip_number_elem(
data, currency_format): # TODO: Support other number systems
continue
for length_elem in currency_format.findall('./currencyFormatLength'):
curr_length_type = length_elem.attrib.get('type')
for elem in length_elem.findall('currencyFormat'):
type = elem.attrib.get('type')
if curr_length_type:
# Handle `<currencyFormatLength type="short">`, etc.
# TODO(3.x): use nested dicts instead of colon-separated madness
type = '%s:%s' % (type, curr_length_type)
if _should_skip_elem(elem, type, currency_formats):
continue
for child in elem.iter():
if child.tag == 'alias':
currency_formats[type] = Alias(
_translate_alias(
['currency_formats', elem.attrib['type']],
child.attrib['path']))
elif child.tag == 'pattern':
pattern = text_type(child.text)
currency_formats[type] = numbers.parse_pattern(pattern)
def parse_decimal_formats(data, tree):
decimal_formats = data.setdefault('decimal_formats', {})
for df_elem in tree.findall('.//decimalFormats'):
if _should_skip_number_elem(
data, df_elem): # TODO: Support other number systems
continue
for elem in df_elem.findall('./decimalFormatLength'):
length_type = elem.attrib.get('type')
if _should_skip_elem(elem, length_type, decimal_formats):
continue
if elem.findall('./alias'):
# TODO map the alias to its target
continue
for pattern_el in elem.findall('./decimalFormat/pattern'):
pattern_type = pattern_el.attrib.get('type')
pattern = numbers.parse_pattern(text_type(pattern_el.text))
if pattern_type:
# This is a compact decimal format, see:
# https://www.unicode.org/reports/tr35/tr35-45/tr35-numbers.html#Compact_Number_Formats
# These are mapped into a `compact_decimal_formats` dictionary
# with the format {length: {count: {multiplier: pattern}}}.
# TODO: Add support for formatting them.
compact_decimal_formats = data.setdefault(
'compact_decimal_formats', {})
length_map = compact_decimal_formats.setdefault(
length_type, {})
length_count_map = length_map.setdefault(
pattern_el.attrib['count'], {})
length_count_map[pattern_type] = pattern
else:
# Regular decimal format.
decimal_formats[length_type] = pattern
def test_parse_pattern_negative():
# No negative format specified
np = numbers.parse_pattern(u'¤#,##0.00')
assert np.prefix == (u'¤', u'-¤')
assert np.suffix == (u'', u'')
# Negative format is specified
np = numbers.parse_pattern(u'¤#,##0.00;(¤#,##0.00)')
assert np.prefix == (u'¤', u'(¤')
assert np.suffix == (u'', u')')
# Negative sign is a suffix
np = numbers.parse_pattern(u'¤ #,##0.00;¤ #,##0.00-')
assert np.prefix == (u'¤ ', u'¤ ')
assert np.suffix == (u'', u'-')
def load_i18n(project_root, tell_sentry):
# Load the locales
localeDir = os.path.join(project_root, 'i18n', 'core')
locales = LOCALES
for file in os.listdir(localeDir):
try:
parts = file.split(".")
if not (len(parts) == 2 and parts[1] == "po"):
continue
lang = parts[0]
with open(os.path.join(localeDir, file)) as f:
l = locales[lang.lower()] = Locale(lang)
c = l.catalog = read_po(f)
c.plural_func = get_function_from_rule(c.plural_expr)
try:
l.countries = make_sorted_dict(COUNTRIES, l.territories)
except KeyError:
l.countries = COUNTRIES
try:
l.languages_2 = make_sorted_dict(LANGUAGES_2, l.languages)
except KeyError:
l.languages_2 = LANGUAGES_2
except Exception as e:
tell_sentry(e, {})
# Add aliases
for k, v in list(locales.items()):
locales.setdefault(ALIASES.get(k, k), v)
locales.setdefault(ALIASES_R.get(k, k), v)
for k, v in list(locales.items()):
locales.setdefault(k.split('_', 1)[0], v)
# Patch the locales to look less formal
locales['fr'].currency_formats[None] = parse_pattern('#,##0.00\u202f\xa4')
locales['fr'].currency_symbols['USD'] = '$'
def parse_currency_formats(data, tree):
currency_formats = data.setdefault('currency_formats', {})
for currency_format in tree.findall('.//currencyFormats'):
if _should_skip_number_elem(data, currency_format): # TODO: Support other number systems
continue
for length_elem in currency_format.findall('./currencyFormatLength'):
curr_length_type = length_elem.attrib.get('type')
for elem in length_elem.findall('currencyFormat'):
type = elem.attrib.get('type')
if curr_length_type:
# Handle `<currencyFormatLength type="short">`, etc.
# TODO(3.x): use nested dicts instead of colon-separated madness
type = '%s:%s' % (type, curr_length_type)
if _should_skip_elem(elem, type, currency_formats):
continue
for child in elem.getiterator():
if child.tag == 'alias':
currency_formats[type] = Alias(
_translate_alias(['currency_formats', elem.attrib['type']],
child.attrib['path'])
)
elif child.tag == 'pattern':
pattern = text_type(child.text)
currency_formats[type] = numbers.parse_pattern(pattern)
def parse_decimal_formats(data, tree):
decimal_formats = data.setdefault('decimal_formats', {})
for df_elem in tree.findall('.//decimalFormats'):
if _should_skip_number_elem(data, df_elem): # TODO: Support other number systems
continue
for elem in df_elem.findall('./decimalFormatLength'):
length_type = elem.attrib.get('type')
if _should_skip_elem(elem, length_type, decimal_formats):
continue
if elem.findall('./alias'):
# TODO map the alias to its target
continue
for pattern_el in elem.findall('./decimalFormat/pattern'):
pattern_type = pattern_el.attrib.get('type')
pattern = numbers.parse_pattern(text_type(pattern_el.text))
if pattern_type:
# This is a compact decimal format, see:
# https://www.unicode.org/reports/tr35/tr35-45/tr35-numbers.html#Compact_Number_Formats
# These are mapped into a `compact_decimal_formats` dictionary
# with the format {length: {count: {multiplier: pattern}}}.
# TODO: Add support for formatting them.
compact_decimal_formats = data.setdefault('compact_decimal_formats', {})
length_map = compact_decimal_formats.setdefault(length_type, {})
length_count_map = length_map.setdefault(pattern_el.attrib['count'], {})
length_count_map[pattern_type] = pattern
else:
# Regular decimal format.
decimal_formats[length_type] = pattern
def test_numberpattern_repr():
"""repr() outputs the pattern string"""
# This implementation looks a bit funny, but that's cause strings are
# repr'd differently in Python 2 vs 3 and this test runs under both.
format = u'¤#,##0.00;(¤#,##0.00)'
np = numbers.parse_pattern(format)
assert repr(format) in repr(np)
def parse_percent_formats(data, tree):
percent_formats = data.setdefault('percent_formats', {})
for elem in tree.findall('.//percentFormats/percentFormatLength'):
type = elem.attrib.get('type')
if _should_skip_elem(elem, type, percent_formats):
continue
pattern = text_type(elem.findtext('percentFormat/pattern'))
percent_formats[type] = numbers.parse_pattern(pattern)
def test_numberpattern_repr():
"""repr() outputs the pattern string"""
# This implementation looks a bit funny, but that's cause strings are
# repr'd differently in Python 2 vs 3 and this test runs under both.
format = u'¤#,##0.00;(¤#,##0.00)'
np = numbers.parse_pattern(format)
assert repr(format) in repr(np)
def test_parse_pattern():
# Original pattern is preserved
np = numbers.parse_pattern(u'¤#,##0.00')
assert np.pattern == u'¤#,##0.00'
np = numbers.parse_pattern(u'¤#,##0.00;(¤#,##0.00)')
assert np.pattern == u'¤#,##0.00;(¤#,##0.00)'
# Given a NumberPattern object, we don't return a new instance.
# However, we don't cache NumberPattern objects, so calling
# parse_pattern with the same format string will create new
# instances
np1 = numbers.parse_pattern(u'¤ #,##0.00')
np2 = numbers.parse_pattern(u'¤ #,##0.00')
assert np1 is not np2
assert np1 is numbers.parse_pattern(np1)
def parse_percent_formats(data, tree):
percent_formats = data.setdefault('percent_formats', {})
for elem in tree.findall('.//percentFormats/percentFormatLength'):
type = elem.attrib.get('type')
if _should_skip_elem(elem, type, percent_formats):
continue
pattern = text_type(elem.findtext('percentFormat/pattern'))
percent_formats[type] = numbers.parse_pattern(pattern)
def test_parse_pattern():
# Original pattern is preserved
np = numbers.parse_pattern(u'¤#,##0.00')
assert np.pattern == u'¤#,##0.00'
np = numbers.parse_pattern(u'¤#,##0.00;(¤#,##0.00)')
assert np.pattern == u'¤#,##0.00;(¤#,##0.00)'
# Given a NumberPattern object, we don't return a new instance.
# However, we don't cache NumberPattern objects, so calling
# parse_pattern with the same format string will create new
# instances
np1 = numbers.parse_pattern(u'¤ #,##0.00')
np2 = numbers.parse_pattern(u'¤ #,##0.00')
assert np1 is not np2
assert np1 is numbers.parse_pattern(np1)
def format_float_field(__, prec, number, locale):
"""Formats a fixed-point field."""
format_ = u'0.'
if prec is None:
format_ += u'#' * NUMBER_DECIMAL_DIGITS
else:
format_ += u'0' * int(prec)
pattern = parse_pattern(format_)
return pattern.apply(number, locale)
def format_float_field(__, prec, number, locale):
"""Formats a fixed-point field."""
format_ = u'0.'
if prec is None:
format_ += u'#' * NUMBER_DECIMAL_DIGITS
else:
format_ += u'0' * int(prec)
pattern = parse_pattern(format_)
return pattern.apply(number, locale)
def format_number(value, digits=None):
locale = get_current_babel_locale()
if digits is None:
return format_decimal(value, locale=locale)
(min_digits, max_digits) = (
digits if isinstance(digits, tuple) else (digits, digits))
format = locale.decimal_formats.get(None)
pattern = parse_pattern(format) # type: babel.numbers.NumberPattern
return pattern.apply(value, locale, force_frac=(min_digits, max_digits))
def parse_decimal_formats(data, tree):
decimal_formats = data.setdefault('decimal_formats', {})
for elem in tree.findall('.//decimalFormats/decimalFormatLength'):
type = elem.attrib.get('type')
if _should_skip_elem(elem, type, decimal_formats):
continue
if elem.findall('./alias'):
# TODO map the alias to its target
continue
pattern = text_type(elem.findtext('./decimalFormat/pattern'))
decimal_formats[type] = numbers.parse_pattern(pattern)
def parse_scientific_formats(data, tree):
scientific_formats = data.setdefault('scientific_formats', {})
for sf_elem in tree.findall('.//scientificFormats'):
if _should_skip_number_elem(data, sf_elem): # TODO: Support other number systems
continue
for elem in sf_elem.findall('./scientificFormatLength'):
type = elem.attrib.get('type')
if _should_skip_elem(elem, type, scientific_formats):
continue
pattern = text_type(elem.findtext('scientificFormat/pattern'))
scientific_formats[type] = numbers.parse_pattern(pattern)
def parse_decimal_formats(data, tree):
decimal_formats = data.setdefault('decimal_formats', {})
for elem in tree.findall('.//decimalFormats/decimalFormatLength'):
type = elem.attrib.get('type')
if _should_skip_elem(elem, type, decimal_formats):
continue
if elem.findall('./alias'):
# TODO map the alias to its target
continue
pattern = text_type(elem.findtext('./decimalFormat/pattern'))
decimal_formats[type] = numbers.parse_pattern(pattern)
def format_field(spec, arg, value, locale):
if spec and isinstance(value, Number):
if arg:
spec += arg
try:
pattern = parse_pattern(spec)
except ValueError:
return spec
else:
return pattern.apply(value, locale)
return str(value)
def format_field(spec, arg, value, locale):
if spec and isinstance(value, Number):
if arg:
spec += arg
try:
pattern = parse_pattern(spec)
except ValueError:
return spec
else:
return pattern.apply(value, locale)
return str(value)
def parse_scientific_formats(data, tree):
scientific_formats = data.setdefault('scientific_formats', {})
for sf_elem in tree.findall('.//scientificFormats'):
if _should_skip_number_elem(
data, sf_elem): # TODO: Support other number systems
continue
for elem in sf_elem.findall('./scientificFormatLength'):
type = elem.attrib.get('type')
if _should_skip_elem(elem, type, scientific_formats):
continue
pattern = text_type(elem.findtext('scientificFormat/pattern'))
scientific_formats[type] = numbers.parse_pattern(pattern)
def processFormats(self, locale, formats):
'''
Process the formats to a complete list of formats that will be used by conversion.
'''
assert isinstance(formats, dict), 'Invalid formats %s' % formats
assert isinstance(locale, Locale), 'Invalid locale %s' % locale
for clsTyp, format in formats.items():
# In here we just check that the format is valid.
try:
if clsTyp in (Number, Percentage): bn.parse_pattern(format)
elif format not in self.formats[clsTyp]: bd.parse_pattern(format)
except Exception as e:
raise FormatError('invalid %s format \'%s\' because: %s' % (clsTyp.__name__, format, str(e)))
if Number not in formats: formats[Number] = locale.decimal_formats.get(None).pattern
if Percentage not in formats: formats[Percentage] = locale.percent_formats.get(None).pattern
for clsTyp, default in self.defaults.items():
if clsTyp not in formats: formats[clsTyp] = default
return formats
def processFormats(self, locale, formats):
'''
Process the formats to a complete list of formats that will be used by conversion.
'''
assert isinstance(formats, dict), 'Invalid formats %s' % formats
assert isinstance(locale, Locale), 'Invalid locale %s' % locale
for clsTyp, format in formats.items():
# In here we just check that the format is valid.
try:
if clsTyp in (Number, Percentage): bn.parse_pattern(format)
elif format not in self.formats[clsTyp]: bd.parse_pattern(format)
except Exception as e:
raise FormatError('invalid %s format \'%s\' because: %s' % (clsTyp.__name__, format, str(e)))
if Number not in formats: formats[Number] = locale.decimal_formats.get(None).pattern
if Percentage not in formats: formats[Percentage] = locale.percent_formats.get(None).pattern
for clsTyp, default in self.defaults.items():
if clsTyp not in formats: formats[clsTyp] = default
return formats
def parse_percent_formats(data, tree):
percent_formats = data.setdefault('percent_formats', {})
for pf_elem in tree.findall('.//percentFormats'):
if _should_skip_number_elem(
data, pf_elem): # TODO: Support other number systems
continue
for elem in pf_elem.findall('.//percentFormatLength'):
type = elem.attrib.get('type')
if _should_skip_elem(elem, type, percent_formats):
continue
pattern = str(elem.findtext('percentFormat/pattern'))
percent_formats[type] = numbers.parse_pattern(pattern)
def load_i18n(project_root, tell_sentry):
# Load the locales
key = lambda t: strip_accents(t[1])
localeDir = os.path.join(project_root, 'i18n', 'core')
locales = i18n.LOCALES
for file in os.listdir(localeDir):
try:
parts = file.split(".")
if not (len(parts) == 2 and parts[1] == "po"):
continue
lang = parts[0]
with open(os.path.join(localeDir, file)) as f:
l = locales[lang.lower()] = Locale(lang)
c = l.catalog = read_po(f)
c.plural_func = get_function_from_rule(c.plural_expr)
try:
l.countries_map = {
k: l.territories[k]
for k in COUNTRIES_MAP
}
l.countries = sorted(l.countries_map.items(), key=key)
except KeyError:
l.countries_map = COUNTRIES_MAP
l.countries = COUNTRIES
except Exception as e:
tell_sentry(e)
# Add the default English locale
locale_en = i18n.LOCALE_EN = locales['en'] = Locale('en')
locale_en.catalog = Catalog('en')
locale_en.catalog.plural_func = lambda n: n != 1
locale_en.countries = COUNTRIES
locale_en.countries_map = COUNTRIES_MAP
# Add aliases
for k, v in list(locales.items()):
locales.setdefault(ALIASES.get(k, k), v)
locales.setdefault(ALIASES_R.get(k, k), v)
for k, v in list(locales.items()):
locales.setdefault(k.split('_', 1)[0], v)
# Patch the locales to look less formal
locales['fr'].currency_formats[None] = parse_pattern('#,##0.00\u202f\xa4')
locales['fr'].currency_symbols['USD'] = '$'
def parse_currency_formats(data, tree):
currency_formats = data.setdefault('currency_formats', {})
for length_elem in tree.findall('.//currencyFormats/currencyFormatLength'):
curr_length_type = length_elem.attrib.get('type')
for elem in length_elem.findall('currencyFormat'):
type = elem.attrib.get('type')
if curr_length_type:
# Handle `<currencyFormatLength type="short">`, etc.
type = '%s:%s' % (type, curr_length_type)
if _should_skip_elem(elem, type, currency_formats):
continue
for child in elem.getiterator():
if child.tag == 'alias':
currency_formats[type] = Alias(
_translate_alias(
['currency_formats', elem.attrib['type']],
child.attrib['path']))
elif child.tag == 'pattern':
pattern = text_type(child.text)
currency_formats[type] = numbers.parse_pattern(pattern)
def parse_currency_formats(data, tree):
currency_formats = data.setdefault('currency_formats', {})
for length_elem in tree.findall('.//currencyFormats/currencyFormatLength'):
curr_length_type = length_elem.attrib.get('type')
for elem in length_elem.findall('currencyFormat'):
type = elem.attrib.get('type')
if curr_length_type:
# Handle `<currencyFormatLength type="short">`, etc.
type = '%s:%s' % (type, curr_length_type)
if _should_skip_elem(elem, type, currency_formats):
continue
for child in elem.getiterator():
if child.tag == 'alias':
currency_formats[type] = Alias(
_translate_alias(['currency_formats', elem.attrib['type']],
child.attrib['path'])
)
elif child.tag == 'pattern':
pattern = text_type(child.text)
currency_formats[type] = numbers.parse_pattern(pattern)
def load_i18n(website):
# Load the locales
key = lambda t: strip_accents(t[1])
localeDir = os.path.join(website.project_root, 'i18n', 'core')
locales = website.locales = {}
for file in os.listdir(localeDir):
try:
parts = file.split(".")
if not (len(parts) == 2 and parts[1] == "po"):
continue
lang = parts[0]
with open(os.path.join(localeDir, file)) as f:
l = locales[lang.lower()] = Locale(lang)
c = l.catalog = read_po(f)
c.plural_func = get_function_from_rule(c.plural_expr)
try:
l.countries_map = {k: l.territories[k] for k in COUNTRIES_MAP}
l.countries = sorted(l.countries_map.items(), key=key)
except KeyError:
l.countries_map = COUNTRIES_MAP
l.countries = COUNTRIES
except Exception as e:
website.tell_sentry(e)
# Add the default English locale
locale_en = website.locale_en = locales['en'] = Locale('en')
locale_en.catalog = Catalog('en')
locale_en.catalog.plural_func = lambda n: n != 1
locale_en.countries = COUNTRIES
locale_en.countries_map = COUNTRIES_MAP
# Add aliases
for k, v in list(locales.items()):
locales.setdefault(ALIASES.get(k, k), v)
locales.setdefault(ALIASES_R.get(k, k), v)
for k, v in list(locales.items()):
locales.setdefault(k.split('_', 1)[0], v)
# Patch the locales to look less formal
locales['fr'].currency_formats[None] = parse_pattern('#,##0.00\u202f\xa4')
locales['fr'].currency_symbols['USD'] = '$'
return number
if isinstance(number, int):
return FluentInt(number, **kwargs)
elif isinstance(number, float):
return FluentFloat(number, **kwargs)
elif isinstance(number, Decimal):
return FluentDecimal(number, **kwargs)
elif isinstance(number, FluentNone):
return number
else:
raise TypeError(
"Can't use fluent_number with object {0} for type {1}".format(
number, type(number)))
_UNGROUPED_PATTERN = parse_pattern("#0")
def clone_pattern(pattern):
return NumberPattern(pattern.pattern, pattern.prefix, pattern.suffix,
pattern.grouping, pattern.int_prec, pattern.frac_prec,
pattern.exp_prec, pattern.exp_plus)
@attr.s
class DateFormatOptions(object):
# Parameters.
# See https://projectfluent.org/fluent/guide/functions.html#datetime
# Developer only
timeZone = attr.ib(default=None)
from unicodedata import combining, normalize
import babel.core
from babel.dates import format_date, format_datetime, format_time, format_timedelta
from babel.messages.pofile import Catalog
from babel.numbers import parse_pattern
from markupsafe import Markup
from pando.utils import utcnow
from ..constants import CURRENCIES, D_MAX
from ..exceptions import AmbiguousNumber, InvalidNumber
from ..website import website
from .currencies import Money, MoneyBasket
MONEY_AMOUNT_FORMAT = parse_pattern('#,##0.00')
ONLY_ZERO = {'0'}
def no_escape(s):
return s
def LegacyMoney(o):
return o if isinstance(o, (Money, MoneyBasket)) else Money(o, 'EUR')
Wrap = namedtuple('Wrap', 'value wrapper')
BOLD = Markup('<b>%s</b>')
def test_parse_static_pattern():
assert numbers.parse_pattern('Kun') # in the So locale in CLDR 30
def load_i18n(canonical_host, canonical_scheme, project_root, tell_sentry):
# Load the locales
localeDir = os.path.join(project_root, 'i18n', 'core')
locales = LOCALES
for file in os.listdir(localeDir):
try:
parts = file.split(".")
if not (len(parts) == 2 and parts[1] == "po"):
continue
lang = parts[0]
with open(os.path.join(localeDir, file)) as f:
l = locales[lang.lower()] = Locale(lang)
c = l.catalog = read_po(f)
c.plural_func = get_function_from_rule(c.plural_expr)
try:
l.countries = make_sorted_dict(COUNTRIES, l.territories)
except KeyError:
l.countries = COUNTRIES
try:
l.languages_2 = make_sorted_dict(LANGUAGES_2, l.languages)
except KeyError:
l.languages_2 = LANGUAGES_2
except Exception as e:
tell_sentry(e, {}, allow_reraise=True)
# Prepare a unique and sorted list for use in the language switcher
percent = lambda l: sum(
(percent(s) if isinstance(s, tuple) else 1) for s in l if s) / len(l)
for l in locales.values():
if l.language == 'en':
l.completion = 1
continue
l.completion = percent([m.string for m in l.catalog if m.id])
loc_url = canonical_scheme + '://%s.' + canonical_host
lang_list = sorted(
((l.completion, l.language, l.language_name.title(),
loc_url % l.language)
for l in set(locales.values()) if l.completion),
key=lambda t: (-t[0], t[1]),
)
# Add aliases
for k, v in list(locales.items()):
locales.setdefault(ALIASES.get(k, k), v)
locales.setdefault(ALIASES_R.get(k, k), v)
for k, v in list(locales.items()):
locales.setdefault(k.split('_', 1)[0], v)
# Patch the locales to look less formal
locales['fr'].currency_formats[None] = parse_pattern('#,##0.00\u202f\xa4')
locales['fr'].currency_symbols['USD'] = '$'
# Load the markdown files
docs = {}
heading_re = re.compile(r'^(#+ )', re.M)
for path in find_files(os.path.join(project_root, 'i18n'), '*.md'):
d, b = os.path.split(path)
doc = os.path.basename(d)
lang = b[:-3]
with open(path, 'rb') as f:
md = f.read().decode('utf8')
if md.startswith('# '):
md = '\n'.join(md.split('\n')[1:]).strip()
md = heading_re.sub(r'##\1', md)
docs.setdefault(doc, {}).__setitem__(lang, markdown.render(md))
return {'docs': docs, 'lang_list': lang_list, 'locales': locales}
def test_parse_pattern():
assert numbers.parse_pattern(u'¤#,##0.00;(¤#,##0.00)').suffix == (u'',
u')')
assert numbers.parse_pattern(u'¤ #,##0.00;¤ #,##0.00-').suffix == (u'',
u'-')
l.countries = COUNTRIES
return langs
# Load the locales
LOCALES = load_langs("i18n")
# Add the default English locale
LOCALE_EN = LOCALES['en'] = Locale('en')
LOCALE_EN.catalog = Catalog('en')
LOCALE_EN.catalog.plural_func = lambda n: n != 1
LOCALE_EN.countries = COUNTRIES
LOCALE_EN.countries_map = COUNTRIES_MAP
# Patch the locales to look less formal
LOCALE_EN.currency_formats[None] = parse_pattern('\xa4#,##0.##')
LOCALES['fr'].currency_formats[None] = parse_pattern('#,##0.##\u202f\xa4')
LOCALES['fr'].currency_symbols['USD'] = '$'
def get_locale_for_request(request):
accept_lang = request.headers.get("Accept-Language", "")
languages = (lang.split(";", 1)[0] for lang in accept_lang.split(","))
for lang in languages:
lang = regularize_locale(lang)
loc = LOCALES.get(lang)
if loc:
return loc
return LOCALE_EN
def _format_currency(cls, number):
locale = Locale.parse(cls.LOCALE)
format = cls.FORMAT or locale.currency_formats.get(None)
pattern = parse_pattern(format)
pattern.frac_prec = (2, cls.DECIMAL_PLACES)
return pattern.apply(number, locale, currency=cls.CODE)
def main():
parser = OptionParser(usage='%prog path/to/cldr')
options, args = parser.parse_args()
if len(args) != 1:
parser.error('incorrect number of arguments')
srcdir = args[0]
destdir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), '..',
'babel')
sup = parse(os.path.join(srcdir, 'supplemental', 'supplementalData.xml'))
# Import global data from the supplemental files
global_data = {}
territory_zones = global_data.setdefault('territory_zones', {})
zone_aliases = global_data.setdefault('zone_aliases', {})
zone_territories = global_data.setdefault('zone_territories', {})
for elem in sup.findall('.//timezoneData/zoneFormatting/zoneItem'):
tzid = elem.attrib['type']
territory_zones.setdefault(elem.attrib['territory'], []).append(tzid)
zone_territories[tzid] = elem.attrib['territory']
if 'aliases' in elem.attrib:
for alias in elem.attrib['aliases'].split():
zone_aliases[alias] = tzid
# Import Metazone mapping
meta_zones = global_data.setdefault('meta_zones', {})
tzsup = parse(os.path.join(srcdir, 'supplemental', 'metazoneInfo.xml'))
for elem in tzsup.findall('.//timezone'):
for child in elem.findall('usesMetazone'):
if 'to' not in child.attrib: # FIXME: support old mappings
meta_zones[elem.attrib['type']] = child.attrib['mzone']
outfile = open(os.path.join(destdir, 'global.dat'), 'wb')
try:
pickle.dump(global_data, outfile, 2)
finally:
outfile.close()
# build a territory containment mapping for inheritance
regions = {}
for elem in sup.findall('.//territoryContainment/group'):
regions[elem.attrib['type']] = elem.attrib['contains'].split()
# Resolve territory containment
territory_containment = {}
region_items = sorted(regions.items())
for group, territory_list in region_items:
for territory in territory_list:
containers = territory_containment.setdefault(territory, set([]))
if group in territory_containment:
containers |= territory_containment[group]
containers.add(group)
# prepare the per-locale plural rules definitions
plural_rules = {}
prsup = parse(os.path.join(srcdir, 'supplemental', 'plurals.xml'))
for elem in prsup.findall('.//plurals/pluralRules'):
rules = []
for rule in elem.findall('pluralRule'):
rules.append((rule.attrib['count'], text_type(rule.text)))
pr = PluralRule(rules)
for locale in elem.attrib['locales'].split():
plural_rules[locale] = pr
filenames = os.listdir(os.path.join(srcdir, 'main'))
filenames.remove('root.xml')
filenames.sort(key=lambda a: len(a))
filenames.insert(0, 'root.xml')
for filename in filenames:
stem, ext = os.path.splitext(filename)
if ext != '.xml':
continue
sys.stderr.write('Processing input file %r\n' % filename)
tree = parse(os.path.join(srcdir, 'main', filename))
data = {}
language = None
elem = tree.find('.//identity/language')
if elem is not None:
language = elem.attrib['type']
sys.stderr.write(' Language: %r\n' % language)
territory = None
elem = tree.find('.//identity/territory')
if elem is not None:
territory = elem.attrib['type']
else:
territory = '001' # world
sys.stderr.write(' Territory: %r\n' % territory)
regions = territory_containment.get(territory, [])
sys.stderr.write(' Regions: %r\n' % regions)
# plural rules
locale_id = '_'.join([
_f for _f in [language, territory != '001' and territory or None]
if _f
])
if locale_id in plural_rules:
data['plural_form'] = plural_rules[locale_id]
# <localeDisplayNames>
territories = data.setdefault('territories', {})
for elem in tree.findall('.//territories/territory'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in territories:
continue
territories[elem.attrib['type']] = _text(elem)
languages = data.setdefault('languages', {})
for elem in tree.findall('.//languages/language'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in languages:
continue
languages[elem.attrib['type']] = _text(elem)
variants = data.setdefault('variants', {})
for elem in tree.findall('.//variants/variant'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in variants:
continue
variants[elem.attrib['type']] = _text(elem)
scripts = data.setdefault('scripts', {})
for elem in tree.findall('.//scripts/script'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in scripts:
continue
scripts[elem.attrib['type']] = _text(elem)
# <dates>
week_data = data.setdefault('week_data', {})
supelem = sup.find('.//weekData')
for elem in supelem.findall('minDays'):
territories = elem.attrib['territories'].split()
if territory in territories or any(
[r in territories for r in regions]):
week_data['min_days'] = int(elem.attrib['count'])
for elem in supelem.findall('firstDay'):
if 'alt' not in elem.attrib: # ignore alternatives
territories = elem.attrib['territories'].split()
if territory in territories or any(
[r in territories for r in regions]):
week_data['first_day'] = weekdays[elem.attrib['day']]
for elem in supelem.findall('weekendStart'):
territories = elem.attrib['territories'].split()
if territory in territories or any(
[r in territories for r in regions]):
week_data['weekend_start'] = weekdays[elem.attrib['day']]
for elem in supelem.findall('weekendEnd'):
territories = elem.attrib['territories'].split()
if territory in territories or any(
[r in territories for r in regions]):
week_data['weekend_end'] = weekdays[elem.attrib['day']]
zone_formats = data.setdefault('zone_formats', {})
for elem in tree.findall('.//timeZoneNames/gmtFormat'):
if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
zone_formats['gmt'] = text_type(elem.text).replace('{0}', '%s')
break
for elem in tree.findall('.//timeZoneNames/regionFormat'):
if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
zone_formats['region'] = text_type(elem.text).replace(
'{0}', '%s')
break
for elem in tree.findall('.//timeZoneNames/fallbackFormat'):
if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
zone_formats['fallback'] = text_type(elem.text) \
.replace('{0}', '%(0)s').replace('{1}', '%(1)s')
break
time_zones = data.setdefault('time_zones', {})
for elem in tree.findall('.//timeZoneNames/zone'):
info = {}
city = elem.findtext('exemplarCity')
if city:
info['city'] = text_type(city)
for child in elem.findall('long/*'):
info.setdefault('long', {})[child.tag] = text_type(child.text)
for child in elem.findall('short/*'):
info.setdefault('short', {})[child.tag] = text_type(child.text)
time_zones[elem.attrib['type']] = info
meta_zones = data.setdefault('meta_zones', {})
for elem in tree.findall('.//timeZoneNames/metazone'):
info = {}
city = elem.findtext('exemplarCity')
if city:
info['city'] = text_type(city)
for child in elem.findall('long/*'):
info.setdefault('long', {})[child.tag] = text_type(child.text)
for child in elem.findall('short/*'):
info.setdefault('short', {})[child.tag] = text_type(child.text)
info['common'] = elem.findtext('commonlyUsed') == 'true'
meta_zones[elem.attrib['type']] = info
for calendar in tree.findall('.//calendars/calendar'):
if calendar.attrib['type'] != 'gregorian':
# TODO: support other calendar types
continue
months = data.setdefault('months', {})
for ctxt in calendar.findall('months/monthContext'):
ctxt_type = ctxt.attrib['type']
ctxts = months.setdefault(ctxt_type, {})
for width in ctxt.findall('monthWidth'):
width_type = width.attrib['type']
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'month':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and int(elem.attrib['type']) in widths:
continue
widths[int(elem.attrib.get('type'))] = text_type(
elem.text)
elif elem.tag == 'alias':
ctxts[width_type] = Alias(
_translate_alias(
['months', ctxt_type, width_type],
elem.attrib['path']))
days = data.setdefault('days', {})
for ctxt in calendar.findall('days/dayContext'):
ctxt_type = ctxt.attrib['type']
ctxts = days.setdefault(ctxt_type, {})
for width in ctxt.findall('dayWidth'):
width_type = width.attrib['type']
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'day':
dtype = weekdays[elem.attrib['type']]
if ('draft' in elem.attrib or 'alt' not in elem.attrib) \
and dtype in widths:
continue
widths[dtype] = text_type(elem.text)
elif elem.tag == 'alias':
ctxts[width_type] = Alias(
_translate_alias(
['days', ctxt_type, width_type],
elem.attrib['path']))
quarters = data.setdefault('quarters', {})
for ctxt in calendar.findall('quarters/quarterContext'):
ctxt_type = ctxt.attrib['type']
ctxts = quarters.setdefault(ctxt.attrib['type'], {})
for width in ctxt.findall('quarterWidth'):
width_type = width.attrib['type']
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'quarter':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and int(elem.attrib['type']) in widths:
continue
widths[int(elem.attrib['type'])] = text_type(
elem.text)
elif elem.tag == 'alias':
ctxts[width_type] = Alias(
_translate_alias(
['quarters', ctxt_type, width_type],
elem.attrib['path']))
eras = data.setdefault('eras', {})
for width in calendar.findall('eras/*'):
width_type = NAME_MAP[width.tag]
widths = eras.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'era':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and int(elem.attrib['type']) in widths:
continue
widths[int(elem.attrib.get('type'))] = text_type(
elem.text)
elif elem.tag == 'alias':
eras[width_type] = Alias(
_translate_alias(['eras', width_type],
elem.attrib['path']))
# AM/PM
periods = data.setdefault('periods', {})
for elem in calendar.findall('am'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.tag in periods:
continue
periods[elem.tag] = text_type(elem.text)
for elem in calendar.findall('pm'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.tag in periods:
continue
periods[elem.tag] = text_type(elem.text)
date_formats = data.setdefault('date_formats', {})
for format in calendar.findall('dateFormats'):
for elem in format.getiterator():
if elem.tag == 'dateFormatLength':
if 'draft' in elem.attrib and \
elem.attrib.get('type') in date_formats:
continue
try:
date_formats[elem.attrib.get('type')] = \
dates.parse_pattern(text_type(elem.findtext('dateFormat/pattern')))
except ValueError:
sys.stderr.write('ERROR: %s\n' % sys.exc_info()[1])
elif elem.tag == 'alias':
date_formats = Alias(
_translate_alias(['date_formats'],
elem.attrib['path']))
time_formats = data.setdefault('time_formats', {})
for format in calendar.findall('timeFormats'):
for elem in format.getiterator():
if elem.tag == 'timeFormatLength':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in time_formats:
continue
try:
time_formats[elem.attrib.get('type')] = \
dates.parse_pattern(text_type(elem.findtext('timeFormat/pattern')))
except ValueError:
sys.stderr.write('ERROR: %s\n' % sys.exc_info()[1])
elif elem.tag == 'alias':
time_formats = Alias(
_translate_alias(['time_formats'],
elem.attrib['path']))
datetime_formats = data.setdefault('datetime_formats', {})
for format in calendar.findall('dateTimeFormats'):
for elem in format.getiterator():
if elem.tag == 'dateTimeFormatLength':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in datetime_formats:
continue
try:
datetime_formats[elem.attrib.get('type')] = \
text_type(elem.findtext('dateTimeFormat/pattern'))
except ValueError:
sys.stderr.write('ERROR: %s\n' % sys.exc_info()[1])
elif elem.tag == 'alias':
datetime_formats = Alias(
_translate_alias(['datetime_formats'],
elem.attrib['path']))
# <numbers>
number_symbols = data.setdefault('number_symbols', {})
for elem in tree.findall('.//numbers/symbols/*'):
if ('draft' in elem.attrib or 'alt' in elem.attrib):
continue
number_symbols[elem.tag] = text_type(elem.text)
decimal_formats = data.setdefault('decimal_formats', {})
for elem in tree.findall('.//decimalFormats/decimalFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in decimal_formats:
continue
pattern = text_type(elem.findtext('decimalFormat/pattern'))
decimal_formats[elem.attrib.get('type')] = numbers.parse_pattern(
pattern)
scientific_formats = data.setdefault('scientific_formats', {})
for elem in tree.findall(
'.//scientificFormats/scientificFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in scientific_formats:
continue
pattern = text_type(elem.findtext('scientificFormat/pattern'))
scientific_formats[elem.attrib.get(
'type')] = numbers.parse_pattern(pattern)
currency_formats = data.setdefault('currency_formats', {})
for elem in tree.findall('.//currencyFormats/currencyFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in currency_formats:
continue
pattern = text_type(elem.findtext('currencyFormat/pattern'))
currency_formats[elem.attrib.get('type')] = numbers.parse_pattern(
pattern)
percent_formats = data.setdefault('percent_formats', {})
for elem in tree.findall('.//percentFormats/percentFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in percent_formats:
continue
pattern = text_type(elem.findtext('percentFormat/pattern'))
percent_formats[elem.attrib.get('type')] = numbers.parse_pattern(
pattern)
currency_names = data.setdefault('currency_names', {})
currency_symbols = data.setdefault('currency_symbols', {})
for elem in tree.findall('.//currencies/currency'):
code = elem.attrib['type']
# TODO: support plural rules for currency name selection
for name in elem.findall('displayName'):
if ('draft' in name.attrib or 'count' in name.attrib) \
and code in currency_names:
continue
currency_names[code] = text_type(name.text)
# TODO: support choice patterns for currency symbol selection
symbol = elem.find('symbol')
if symbol is not None and 'draft' not in symbol.attrib \
and 'choice' not in symbol.attrib:
currency_symbols[code] = text_type(symbol.text)
# <units>
unit_patterns = data.setdefault('unit_patterns', {})
for elem in tree.findall('.//units/unit'):
unit_type = elem.attrib['type']
unit_pattern = unit_patterns.setdefault(unit_type, {})
for pattern in elem.findall('unitPattern'):
unit_patterns[unit_type][pattern.attrib['count']] = \
text_type(pattern.text)
outfile = open(os.path.join(destdir, 'localedata', stem + '.dat'),
'wb')
try:
pickle.dump(data, outfile, 2)
finally:
outfile.close()
def test_parse_pattern():
assert numbers.parse_pattern(u'¤#,##0.00;(¤#,##0.00)').suffix == (u'', u')')
assert numbers.parse_pattern(u'¤ #,##0.00;¤ #,##0.00-').suffix == (u'', u'-')
l.countries = COUNTRIES
return langs
# Load the locales
LOCALES = load_langs("i18n")
# Add the default English locale
LOCALE_EN = LOCALES['en'] = Locale('en')
LOCALE_EN.catalog = Catalog('en')
LOCALE_EN.catalog.plural_func = lambda n: n != 1
LOCALE_EN.countries = COUNTRIES
LOCALE_EN.countries_map = COUNTRIES_MAP
# Patch the locales to look less formal
LOCALES['fr'].currency_formats[None] = parse_pattern('#,##0.00\u202f\xa4')
LOCALES['fr'].currency_symbols['USD'] = '$'
def get_locale_for_request(request):
accept_lang = request.headers.get("Accept-Language", "")
languages = (lang.split(";", 1)[0] for lang in accept_lang.split(","))
for lang in languages:
lang = regularize_locale(lang)
loc = LOCALES.get(lang)
if loc:
return loc
return LOCALE_EN
def format_currency_with_options(number,
def main():
parser = OptionParser(usage='%prog path/to/cldr')
options, args = parser.parse_args()
if len(args) != 1:
parser.error('incorrect number of arguments')
srcdir = args[0]
destdir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),
'..', 'babel')
sup = parse(os.path.join(srcdir, 'supplemental', 'supplementalData.xml'))
# Import global data from the supplemental files
global_data = {}
territory_zones = global_data.setdefault('territory_zones', {})
zone_aliases = global_data.setdefault('zone_aliases', {})
zone_territories = global_data.setdefault('zone_territories', {})
for elem in sup.findall('.//timezoneData/zoneFormatting/zoneItem'):
tzid = elem.attrib['type']
territory_zones.setdefault(elem.attrib['territory'], []).append(tzid)
zone_territories[tzid] = elem.attrib['territory']
if 'aliases' in elem.attrib:
for alias in elem.attrib['aliases'].split():
zone_aliases[alias] = tzid
# Import Metazone mapping
meta_zones = global_data.setdefault('meta_zones', {})
tzsup = parse(os.path.join(srcdir, 'supplemental', 'metazoneInfo.xml'))
for elem in tzsup.findall('.//timezone'):
for child in elem.findall('usesMetazone'):
if 'to' not in child.attrib: # FIXME: support old mappings
meta_zones[elem.attrib['type']] = child.attrib['mzone']
outfile = open(os.path.join(destdir, 'global.dat'), 'wb')
try:
pickle.dump(global_data, outfile, 2)
finally:
outfile.close()
# build a territory containment mapping for inheritance
regions = {}
for elem in sup.findall('.//territoryContainment/group'):
regions[elem.attrib['type']] = elem.attrib['contains'].split()
# Resolve territory containment
territory_containment = {}
region_items = sorted(regions.items())
for group, territory_list in region_items:
for territory in territory_list:
containers = territory_containment.setdefault(territory, set([]))
if group in territory_containment:
containers |= territory_containment[group]
containers.add(group)
# prepare the per-locale plural rules definitions
plural_rules = {}
prsup = parse(os.path.join(srcdir, 'supplemental', 'plurals.xml'))
for elem in prsup.findall('.//plurals/pluralRules'):
rules = []
for rule in elem.findall('pluralRule'):
rules.append((rule.attrib['count'], text_type(rule.text)))
pr = PluralRule(rules)
for locale in elem.attrib['locales'].split():
plural_rules[locale] = pr
filenames = os.listdir(os.path.join(srcdir, 'main'))
filenames.remove('root.xml')
filenames.sort(key=lambda a: len(a))
filenames.insert(0, 'root.xml')
for filename in filenames:
stem, ext = os.path.splitext(filename)
if ext != '.xml':
continue
sys.stderr.write('Processing input file %r\n' % filename)
tree = parse(os.path.join(srcdir, 'main', filename))
data = {}
language = None
elem = tree.find('.//identity/language')
if elem is not None:
language = elem.attrib['type']
sys.stderr.write(' Language: %r\n' % language)
territory = None
elem = tree.find('.//identity/territory')
if elem is not None:
territory = elem.attrib['type']
else:
territory = '001' # world
sys.stderr.write(' Territory: %r\n' % territory)
regions = territory_containment.get(territory, [])
sys.stderr.write(' Regions: %r\n' % regions)
# plural rules
locale_id = '_'.join([_f for _f in [
language,
territory != '001' and territory or None
] if _f])
if locale_id in plural_rules:
data['plural_form'] = plural_rules[locale_id]
# <localeDisplayNames>
territories = data.setdefault('territories', {})
for elem in tree.findall('.//territories/territory'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in territories:
continue
territories[elem.attrib['type']] = _text(elem)
languages = data.setdefault('languages', {})
for elem in tree.findall('.//languages/language'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in languages:
continue
languages[elem.attrib['type']] = _text(elem)
variants = data.setdefault('variants', {})
for elem in tree.findall('.//variants/variant'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in variants:
continue
variants[elem.attrib['type']] = _text(elem)
scripts = data.setdefault('scripts', {})
for elem in tree.findall('.//scripts/script'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in scripts:
continue
scripts[elem.attrib['type']] = _text(elem)
# <dates>
week_data = data.setdefault('week_data', {})
supelem = sup.find('.//weekData')
for elem in supelem.findall('minDays'):
territories = elem.attrib['territories'].split()
if territory in territories or any([r in territories for r in regions]):
week_data['min_days'] = int(elem.attrib['count'])
for elem in supelem.findall('firstDay'):
territories = elem.attrib['territories'].split()
if territory in territories or any([r in territories for r in regions]):
week_data['first_day'] = weekdays[elem.attrib['day']]
for elem in supelem.findall('weekendStart'):
territories = elem.attrib['territories'].split()
if territory in territories or any([r in territories for r in regions]):
week_data['weekend_start'] = weekdays[elem.attrib['day']]
for elem in supelem.findall('weekendEnd'):
territories = elem.attrib['territories'].split()
if territory in territories or any([r in territories for r in regions]):
week_data['weekend_end'] = weekdays[elem.attrib['day']]
zone_formats = data.setdefault('zone_formats', {})
for elem in tree.findall('.//timeZoneNames/gmtFormat'):
if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
zone_formats['gmt'] = text_type(elem.text).replace('{0}', '%s')
break
for elem in tree.findall('.//timeZoneNames/regionFormat'):
if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
zone_formats['region'] = text_type(elem.text).replace('{0}', '%s')
break
for elem in tree.findall('.//timeZoneNames/fallbackFormat'):
if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
zone_formats['fallback'] = text_type(elem.text) \
.replace('{0}', '%(0)s').replace('{1}', '%(1)s')
break
time_zones = data.setdefault('time_zones', {})
for elem in tree.findall('.//timeZoneNames/zone'):
info = {}
city = elem.findtext('exemplarCity')
if city:
info['city'] = text_type(city)
for child in elem.findall('long/*'):
info.setdefault('long', {})[child.tag] = text_type(child.text)
for child in elem.findall('short/*'):
info.setdefault('short', {})[child.tag] = text_type(child.text)
time_zones[elem.attrib['type']] = info
meta_zones = data.setdefault('meta_zones', {})
for elem in tree.findall('.//timeZoneNames/metazone'):
info = {}
city = elem.findtext('exemplarCity')
if city:
info['city'] = text_type(city)
for child in elem.findall('long/*'):
info.setdefault('long', {})[child.tag] = text_type(child.text)
for child in elem.findall('short/*'):
info.setdefault('short', {})[child.tag] = text_type(child.text)
info['common'] = elem.findtext('commonlyUsed') == 'true'
meta_zones[elem.attrib['type']] = info
for calendar in tree.findall('.//calendars/calendar'):
if calendar.attrib['type'] != 'gregorian':
# TODO: support other calendar types
continue
months = data.setdefault('months', {})
for ctxt in calendar.findall('months/monthContext'):
ctxt_type = ctxt.attrib['type']
ctxts = months.setdefault(ctxt_type, {})
for width in ctxt.findall('monthWidth'):
width_type = width.attrib['type']
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'month':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and int(elem.attrib['type']) in widths:
continue
widths[int(elem.attrib.get('type'))] = text_type(elem.text)
elif elem.tag == 'alias':
ctxts[width_type] = Alias(
_translate_alias(['months', ctxt_type, width_type],
elem.attrib['path'])
)
days = data.setdefault('days', {})
for ctxt in calendar.findall('days/dayContext'):
ctxt_type = ctxt.attrib['type']
ctxts = days.setdefault(ctxt_type, {})
for width in ctxt.findall('dayWidth'):
width_type = width.attrib['type']
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'day':
dtype = weekdays[elem.attrib['type']]
if ('draft' in elem.attrib or 'alt' not in elem.attrib) \
and dtype in widths:
continue
widths[dtype] = text_type(elem.text)
elif elem.tag == 'alias':
ctxts[width_type] = Alias(
_translate_alias(['days', ctxt_type, width_type],
elem.attrib['path'])
)
quarters = data.setdefault('quarters', {})
for ctxt in calendar.findall('quarters/quarterContext'):
ctxt_type = ctxt.attrib['type']
ctxts = quarters.setdefault(ctxt.attrib['type'], {})
for width in ctxt.findall('quarterWidth'):
width_type = width.attrib['type']
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'quarter':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and int(elem.attrib['type']) in widths:
continue
widths[int(elem.attrib['type'])] = text_type(elem.text)
elif elem.tag == 'alias':
ctxts[width_type] = Alias(
_translate_alias(['quarters', ctxt_type, width_type],
elem.attrib['path'])
)
eras = data.setdefault('eras', {})
for width in calendar.findall('eras/*'):
width_type = NAME_MAP[width.tag]
widths = eras.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'era':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and int(elem.attrib['type']) in widths:
continue
widths[int(elem.attrib.get('type'))] = text_type(elem.text)
elif elem.tag == 'alias':
eras[width_type] = Alias(
_translate_alias(['eras', width_type],
elem.attrib['path'])
)
# AM/PM
periods = data.setdefault('periods', {})
for elem in calendar.findall('am'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.tag in periods:
continue
periods[elem.tag] = text_type(elem.text)
for elem in calendar.findall('pm'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.tag in periods:
continue
periods[elem.tag] = text_type(elem.text)
date_formats = data.setdefault('date_formats', {})
for format in calendar.findall('dateFormats'):
for elem in format.getiterator():
if elem.tag == 'dateFormatLength':
if 'draft' in elem.attrib and \
elem.attrib.get('type') in date_formats:
continue
try:
date_formats[elem.attrib.get('type')] = \
dates.parse_pattern(text_type(elem.findtext('dateFormat/pattern')))
except ValueError:
sys.stderr.write('ERROR: %s\n' % sys.exc_info()[1])
elif elem.tag == 'alias':
date_formats = Alias(_translate_alias(
['date_formats'], elem.attrib['path'])
)
time_formats = data.setdefault('time_formats', {})
for format in calendar.findall('timeFormats'):
for elem in format.getiterator():
if elem.tag == 'timeFormatLength':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in time_formats:
continue
try:
time_formats[elem.attrib.get('type')] = \
dates.parse_pattern(text_type(elem.findtext('timeFormat/pattern')))
except ValueError:
sys.stderr.write('ERROR: %s\n' % sys.exc_info()[1])
elif elem.tag == 'alias':
time_formats = Alias(_translate_alias(
['time_formats'], elem.attrib['path'])
)
datetime_formats = data.setdefault('datetime_formats', {})
for format in calendar.findall('dateTimeFormats'):
for elem in format.getiterator():
if elem.tag == 'dateTimeFormatLength':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in datetime_formats:
continue
try:
datetime_formats[elem.attrib.get('type')] = \
text_type(elem.findtext('dateTimeFormat/pattern'))
except ValueError:
sys.stderr.write('ERROR: %s\n' % sys.exc_info()[1])
elif elem.tag == 'alias':
datetime_formats = Alias(_translate_alias(
['datetime_formats'], elem.attrib['path'])
)
# <numbers>
number_symbols = data.setdefault('number_symbols', {})
for elem in tree.findall('.//numbers/symbols/*'):
if ('draft' in elem.attrib or 'alt' in elem.attrib):
continue
number_symbols[elem.tag] = text_type(elem.text)
decimal_formats = data.setdefault('decimal_formats', {})
for elem in tree.findall('.//decimalFormats/decimalFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in decimal_formats:
continue
pattern = text_type(elem.findtext('decimalFormat/pattern'))
decimal_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern)
scientific_formats = data.setdefault('scientific_formats', {})
for elem in tree.findall('.//scientificFormats/scientificFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in scientific_formats:
continue
pattern = text_type(elem.findtext('scientificFormat/pattern'))
scientific_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern)
currency_formats = data.setdefault('currency_formats', {})
for elem in tree.findall('.//currencyFormats/currencyFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in currency_formats:
continue
pattern = text_type(elem.findtext('currencyFormat/pattern'))
currency_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern)
percent_formats = data.setdefault('percent_formats', {})
for elem in tree.findall('.//percentFormats/percentFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in percent_formats:
continue
pattern = text_type(elem.findtext('percentFormat/pattern'))
percent_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern)
currency_names = data.setdefault('currency_names', {})
currency_symbols = data.setdefault('currency_symbols', {})
for elem in tree.findall('.//currencies/currency'):
code = elem.attrib['type']
# TODO: support plural rules for currency name selection
for name in elem.findall('displayName'):
if ('draft' in name.attrib or 'count' in name.attrib) \
and code in currency_names:
continue
currency_names[code] = text_type(name.text)
# TODO: support choice patterns for currency symbol selection
symbol = elem.find('symbol')
if symbol is not None and 'draft' not in symbol.attrib \
and 'choice' not in symbol.attrib:
currency_symbols[code] = text_type(symbol.text)
# <units>
unit_patterns = data.setdefault('unit_patterns', {})
for elem in tree.findall('.//units/unit'):
unit_type = elem.attrib['type']
unit_pattern = unit_patterns.setdefault(unit_type, {})
for pattern in elem.findall('unitPattern'):
unit_patterns[unit_type][pattern.attrib['count']] = \
text_type(pattern.text)
dest = os.path.join(destdir, 'localedata', stem + '.dat')
if not os.path.exists(os.path.dirname(dest)):
os.mkdir(os.path.dirname(dest))
outfile = open(dest, 'wb')
try:
pickle.dump(data, outfile, 2)
finally:
outfile.close()
def load_i18n(canonical_host, canonical_scheme, project_root, tell_sentry):
# Load the locales
localeDir = os.path.join(project_root, 'i18n', 'core')
locales = LOCALES
source_strings = {}
for file in os.listdir(localeDir):
try:
parts = file.split(".")
if not (len(parts) == 2 and parts[1] == "po"):
continue
lang = parts[0]
with open(os.path.join(localeDir, file), 'rb') as f:
l = locales[lang.lower()] = Locale(lang)
c = l.catalog = read_po(f)
share_source_strings(c, source_strings)
c.plural_func = get_function_from_rule(c.plural_expr)
replace_unused_singulars(c)
try:
l.countries = make_sorted_dict(COUNTRIES, l.territories)
except KeyError:
l.countries = COUNTRIES
try:
l.languages_2 = make_sorted_dict(LANGUAGES_2, l.languages)
except KeyError:
l.languages_2 = LANGUAGES_2
except Exception as e:
tell_sentry(e, {})
del source_strings
# Prepare a unique and sorted list for use in the language switcher
percent = lambda l, total: sum((percent(s, len(s)) if isinstance(s, tuple) else 1) for s in l if s) / total
for l in list(locales.values()):
if l.language == 'en':
l.completion = 1
continue
l.completion = percent([m.string for m in l.catalog if m.id and not m.fuzzy], len(l.catalog))
if l.completion == 0:
del locales[l.language]
loc_url = canonical_scheme+'://%s.'+canonical_host
domain, port = (canonical_host.split(':') + [None])[:2]
port = int(port) if port else socket.getservbyname(canonical_scheme, 'tcp')
subdomains = {
l.subdomain: loc_url % l.subdomain for l in locales.values()
if resolve(l.subdomain + '.' + domain, port)
}
lang_list = sorted(
(
(l.completion, l.language, l.language_name.title(), loc_url % l.subdomain)
for l in set(locales.values()) if l.completion > 0.5
),
key=lambda t: (-t[0], t[1]),
)
# Add year-less date format
year_re = re.compile(r'(^y+[^a-zA-Z]+|[^a-zA-Z]+y+$)')
for l in locales.values():
short_format = l.date_formats['short'].pattern
assert short_format[0] == 'y' or short_format[-1] == 'y', (l.language, short_format)
l.date_formats['short_yearless'] = year_re.sub('', short_format)
# Add aliases
for k, v in list(locales.items()):
locales.setdefault(ALIASES.get(k, k), v)
locales.setdefault(ALIASES_R.get(k, k), v)
for k, v in list(locales.items()):
locales.setdefault(k.split('_', 1)[0], v)
# Patch the locales to look less formal
locales['fr'].currency_formats['standard'] = parse_pattern('#,##0.00\u202f\xa4')
locales['fr'].currencies['USD'] = 'dollar états-unien'
# Load the markdown files
docs = {}
heading_re = re.compile(r'^(#+ )', re.M)
for path in find_files(os.path.join(project_root, 'i18n'), '*.md'):
d, b = os.path.split(path)
doc = os.path.basename(d)
lang = b[:-3]
with open(path, 'rb') as f:
md = f.read().decode('utf8')
if md.startswith('# '):
md = '\n'.join(md.split('\n')[1:]).strip()
md = heading_re.sub(r'##\1', md)
docs.setdefault(doc, {}).__setitem__(lang, markdown.render(md))
return {'docs': docs, 'lang_list': lang_list, 'locales': locales, 'subdomains': subdomains}
# <numbers>
number_symbols = data.setdefault('number_symbols', {})
for elem in tree.findall('.//numbers/symbols/*'):
if ('draft' in elem.attrib or 'alt' in elem.attrib):
continue
number_symbols[elem.tag] = unicode(elem.text)
decimal_formats = data.setdefault('decimal_formats', {})
for elem in tree.findall('.//decimalFormats/decimalFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in decimal_formats:
continue
pattern = unicode(elem.findtext('decimalFormat/pattern'))
decimal_formats[elem.attrib.get('type')] = numbers.parse_pattern(
pattern)
scientific_formats = data.setdefault('scientific_formats', {})
for elem in tree.findall(
'.//scientificFormats/scientificFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in scientific_formats:
continue
pattern = unicode(elem.findtext('scientificFormat/pattern'))
scientific_formats[elem.attrib.get(
'type')] = numbers.parse_pattern(pattern)
currency_formats = data.setdefault('currency_formats', {})
for elem in tree.findall('.//currencyFormats/currencyFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in currency_formats:
l.countries = COUNTRIES
return langs
# Load the locales
LOCALES = load_langs("i18n/core")
# Add the default English locale
LOCALE_EN = LOCALES['en'] = Locale('en')
LOCALE_EN.catalog = Catalog('en')
LOCALE_EN.catalog.plural_func = lambda n: n != 1
LOCALE_EN.countries = COUNTRIES
LOCALE_EN.countries_map = COUNTRIES_MAP
# Patch the locales to look less formal
LOCALES['fr'].currency_formats[None] = parse_pattern('#,##0.00\u202f\xa4')
LOCALES['fr'].currency_symbols['USD'] = '$'
def get_locale_for_request(request):
accept_lang = request.headers.get("Accept-Language", "")
languages = (lang.split(";", 1)[0] for lang in accept_lang.split(","))
for lang in languages:
lang = regularize_locale(lang)
loc = LOCALES.get(lang)
if loc:
return loc
return LOCALE_EN
def format_currency_with_options(number, currency, locale=LOCALE_EN, trailing_zeroes=True):
def load_i18n(canonical_host, canonical_scheme, project_root, tell_sentry):
# Load the locales
localeDir = os.path.join(project_root, 'i18n', 'core')
locales = LOCALES
for file in os.listdir(localeDir):
try:
parts = file.split(".")
if not (len(parts) == 2 and parts[1] == "po"):
continue
lang = parts[0]
with open(os.path.join(localeDir, file)) as f:
l = locales[lang.lower()] = Locale(lang)
c = l.catalog = read_po(f)
c.plural_func = get_function_from_rule(c.plural_expr)
try:
l.countries = make_sorted_dict(COUNTRIES, l.territories)
except KeyError:
l.countries = COUNTRIES
try:
l.languages_2 = make_sorted_dict(LANGUAGES_2, l.languages)
except KeyError:
l.languages_2 = LANGUAGES_2
except Exception as e:
tell_sentry(e, {}, allow_reraise=True)
# Prepare a unique and sorted list for use in the language switcher
for l in locales.values():
strings = [m.string for m in l.catalog]
l.completion = sum(1 for s in strings if s) / len(strings)
loc_url = canonical_scheme+'://%s.'+canonical_host
lang_list = sorted(
(
(l.completion, l.language, l.language_name.title(), loc_url % l.language)
for l in set(locales.values())
),
key=lambda t: (-t[0], t[1]),
)
# Add aliases
for k, v in list(locales.items()):
locales.setdefault(ALIASES.get(k, k), v)
locales.setdefault(ALIASES_R.get(k, k), v)
for k, v in list(locales.items()):
locales.setdefault(k.split('_', 1)[0], v)
# Patch the locales to look less formal
locales['fr'].currency_formats[None] = parse_pattern('#,##0.00\u202f\xa4')
locales['fr'].currency_symbols['USD'] = '$'
# Load the markdown files
docs = {}
heading_re = re.compile(r'^(#+ )', re.M)
for path in find_files(os.path.join(project_root, 'i18n'), '*.md'):
d, b = os.path.split(path)
doc = os.path.basename(d)
lang = b[:-3]
with open(path, 'rb') as f:
md = f.read().decode('utf8')
if md.startswith('# '):
md = '\n'.join(md.split('\n')[1:]).strip()
md = heading_re.sub(r'##\1', md)
docs.setdefault(doc, {}).__setitem__(lang, markdown.render(md))
return {'docs': docs, 'lang_list': lang_list, 'locales': locales}
def main():
parser = OptionParser(usage='%prog path/to/cldr')
options, args = parser.parse_args()
if len(args) != 1:
parser.error('incorrect number of arguments')
srcdir = args[0]
destdir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),
'..', 'babel')
sup_filename = os.path.join(srcdir, 'supplemental', 'supplementalData.xml')
bcp47_timezone = parse(os.path.join(srcdir, 'bcp47', 'timezone.xml'))
sup_windows_zones = parse(os.path.join(srcdir, 'supplemental',
'windowsZones.xml'))
sup_metadata = parse(os.path.join(srcdir, 'supplemental',
'supplementalMetadata.xml'))
sup_likely = parse(os.path.join(srcdir, 'supplemental',
'likelySubtags.xml'))
sup = parse(sup_filename)
# Import global data from the supplemental files
global_path = os.path.join(destdir, 'global.dat')
global_data = {}
if need_conversion(global_path, global_data, sup_filename):
territory_zones = global_data.setdefault('territory_zones', {})
zone_aliases = global_data.setdefault('zone_aliases', {})
zone_territories = global_data.setdefault('zone_territories', {})
win_mapping = global_data.setdefault('windows_zone_mapping', {})
language_aliases = global_data.setdefault('language_aliases', {})
territory_aliases = global_data.setdefault('territory_aliases', {})
script_aliases = global_data.setdefault('script_aliases', {})
variant_aliases = global_data.setdefault('variant_aliases', {})
likely_subtags = global_data.setdefault('likely_subtags', {})
territory_currencies = global_data.setdefault('territory_currencies', {})
parent_exceptions = global_data.setdefault('parent_exceptions', {})
# create auxiliary zone->territory map from the windows zones (we don't set
# the 'zones_territories' map directly here, because there are some zones
# aliases listed and we defer the decision of which ones to choose to the
# 'bcp47' data
_zone_territory_map = {}
for map_zone in sup_windows_zones.findall(
'.//windowsZones/mapTimezones/mapZone'):
if map_zone.attrib.get('territory') == '001':
win_mapping[map_zone.attrib['other']] = \
map_zone.attrib['type'].split()[0]
for tzid in text_type(map_zone.attrib['type']).split():
_zone_territory_map[tzid] = \
text_type(map_zone.attrib['territory'])
for key_elem in bcp47_timezone.findall('.//keyword/key'):
if key_elem.attrib['name'] == 'tz':
for elem in key_elem.findall('type'):
if 'deprecated' not in elem.attrib:
aliases = text_type(elem.attrib['alias']).split()
tzid = aliases.pop(0)
territory = _zone_territory_map.get(tzid, '001')
territory_zones.setdefault(territory, []).append(tzid)
zone_territories[tzid] = territory
for alias in aliases:
zone_aliases[alias] = tzid
break
# Import Metazone mapping
meta_zones = global_data.setdefault('meta_zones', {})
tzsup = parse(os.path.join(srcdir, 'supplemental', 'metaZones.xml'))
for elem in tzsup.findall('.//timezone'):
for child in elem.findall('usesMetazone'):
if 'to' not in child.attrib: # FIXME: support old mappings
meta_zones[elem.attrib['type']] = child.attrib['mzone']
# Language aliases
for alias in sup_metadata.findall('.//alias/languageAlias'):
# We don't have a use for those at the moment. They don't
# pass our parser anyways.
if '_' in alias.attrib['type']:
continue
language_aliases[alias.attrib['type']] = alias.attrib['replacement']
# Territory aliases
for alias in sup_metadata.findall('.//alias/territoryAlias'):
territory_aliases[alias.attrib['type']] = \
alias.attrib['replacement'].split()
# Script aliases
for alias in sup_metadata.findall('.//alias/scriptAlias'):
script_aliases[alias.attrib['type']] = alias.attrib['replacement']
# Variant aliases
for alias in sup_metadata.findall('.//alias/variantAlias'):
repl = alias.attrib.get('replacement')
if repl:
variant_aliases[alias.attrib['type']] = repl
# Likely subtags
for likely_subtag in sup_likely.findall('.//likelySubtags/likelySubtag'):
likely_subtags[likely_subtag.attrib['from']] = \
likely_subtag.attrib['to']
# Currencies in territories
for region in sup.findall('.//currencyData/region'):
region_code = region.attrib['iso3166']
region_currencies = []
for currency in region.findall('./currency'):
cur_start = _parse_currency_date(currency.attrib.get('from'))
cur_end = _parse_currency_date(currency.attrib.get('to'))
region_currencies.append((currency.attrib['iso4217'],
cur_start, cur_end,
currency.attrib.get(
'tender', 'true') == 'true'))
region_currencies.sort(key=_currency_sort_key)
territory_currencies[region_code] = region_currencies
# Explicit parent locales
for paternity in sup.findall('.//parentLocales/parentLocale'):
parent = paternity.attrib['parent']
for child in paternity.attrib['locales'].split():
parent_exceptions[child] = parent
outfile = open(global_path, 'wb')
try:
pickle.dump(global_data, outfile, 2)
finally:
outfile.close()
# build a territory containment mapping for inheritance
regions = {}
for elem in sup.findall('.//territoryContainment/group'):
regions[elem.attrib['type']] = elem.attrib['contains'].split()
# Resolve territory containment
territory_containment = {}
region_items = sorted(regions.items())
for group, territory_list in region_items:
for territory in territory_list:
containers = territory_containment.setdefault(territory, set([]))
if group in territory_containment:
containers |= territory_containment[group]
containers.add(group)
# prepare the per-locale plural rules definitions
plural_rules = {}
prsup = parse(os.path.join(srcdir, 'supplemental', 'plurals.xml'))
for elem in prsup.findall('.//plurals/pluralRules'):
rules = []
for rule in elem.findall('pluralRule'):
rules.append((rule.attrib['count'], text_type(rule.text)))
pr = PluralRule(rules)
for locale in elem.attrib['locales'].split():
plural_rules[locale] = pr
filenames = os.listdir(os.path.join(srcdir, 'main'))
filenames.remove('root.xml')
filenames.sort(key=len)
filenames.insert(0, 'root.xml')
for filename in filenames:
stem, ext = os.path.splitext(filename)
if ext != '.xml':
continue
full_filename = os.path.join(srcdir, 'main', filename)
data_filename = os.path.join(destdir, 'localedata', stem + '.dat')
data = {}
if not need_conversion(data_filename, data, full_filename):
continue
tree = parse(full_filename)
language = None
elem = tree.find('.//identity/language')
if elem is not None:
language = elem.attrib['type']
territory = None
elem = tree.find('.//identity/territory')
if elem is not None:
territory = elem.attrib['type']
else:
territory = '001' # world
regions = territory_containment.get(territory, [])
log('Processing %s (Language = %s; Territory = %s)',
filename, language, territory)
# plural rules
locale_id = '_'.join(filter(None, [
language,
territory != '001' and territory or None
]))
if locale_id in plural_rules:
data['plural_form'] = plural_rules[locale_id]
# <localeDisplayNames>
territories = data.setdefault('territories', {})
for elem in tree.findall('.//territories/territory'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in territories:
continue
territories[elem.attrib['type']] = _text(elem)
languages = data.setdefault('languages', {})
for elem in tree.findall('.//languages/language'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in languages:
continue
languages[elem.attrib['type']] = _text(elem)
variants = data.setdefault('variants', {})
for elem in tree.findall('.//variants/variant'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in variants:
continue
variants[elem.attrib['type']] = _text(elem)
scripts = data.setdefault('scripts', {})
for elem in tree.findall('.//scripts/script'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in scripts:
continue
scripts[elem.attrib['type']] = _text(elem)
# <dates>
week_data = data.setdefault('week_data', {})
supelem = sup.find('.//weekData')
for elem in supelem.findall('minDays'):
territories = elem.attrib['territories'].split()
if territory in territories or any([r in territories for r in regions]):
week_data['min_days'] = int(elem.attrib['count'])
for elem in supelem.findall('firstDay'):
territories = elem.attrib['territories'].split()
if territory in territories or any([r in territories for r in regions]):
week_data['first_day'] = weekdays[elem.attrib['day']]
for elem in supelem.findall('weekendStart'):
territories = elem.attrib['territories'].split()
if territory in territories or any([r in territories for r in regions]):
week_data['weekend_start'] = weekdays[elem.attrib['day']]
for elem in supelem.findall('weekendEnd'):
territories = elem.attrib['territories'].split()
if territory in territories or any([r in territories for r in regions]):
week_data['weekend_end'] = weekdays[elem.attrib['day']]
zone_formats = data.setdefault('zone_formats', {})
for elem in tree.findall('.//timeZoneNames/gmtFormat'):
if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
zone_formats['gmt'] = text_type(elem.text).replace('{0}', '%s')
break
for elem in tree.findall('.//timeZoneNames/regionFormat'):
if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
zone_formats['region'] = text_type(elem.text).replace('{0}', '%s')
break
for elem in tree.findall('.//timeZoneNames/fallbackFormat'):
if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
zone_formats['fallback'] = text_type(elem.text) \
.replace('{0}', '%(0)s').replace('{1}', '%(1)s')
break
for elem in tree.findall('.//timeZoneNames/fallbackRegionFormat'):
if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
zone_formats['fallback_region'] = text_type(elem.text) \
.replace('{0}', '%(0)s').replace('{1}', '%(1)s')
break
time_zones = data.setdefault('time_zones', {})
for elem in tree.findall('.//timeZoneNames/zone'):
info = {}
city = elem.findtext('exemplarCity')
if city:
info['city'] = text_type(city)
for child in elem.findall('long/*'):
info.setdefault('long', {})[child.tag] = text_type(child.text)
for child in elem.findall('short/*'):
info.setdefault('short', {})[child.tag] = text_type(child.text)
time_zones[elem.attrib['type']] = info
meta_zones = data.setdefault('meta_zones', {})
for elem in tree.findall('.//timeZoneNames/metazone'):
info = {}
city = elem.findtext('exemplarCity')
if city:
info['city'] = text_type(city)
for child in elem.findall('long/*'):
info.setdefault('long', {})[child.tag] = text_type(child.text)
for child in elem.findall('short/*'):
info.setdefault('short', {})[child.tag] = text_type(child.text)
meta_zones[elem.attrib['type']] = info
for calendar in tree.findall('.//calendars/calendar'):
if calendar.attrib['type'] != 'gregorian':
# TODO: support other calendar types
continue
months = data.setdefault('months', {})
for ctxt in calendar.findall('months/monthContext'):
ctxt_type = ctxt.attrib['type']
ctxts = months.setdefault(ctxt_type, {})
for width in ctxt.findall('monthWidth'):
width_type = width.attrib['type']
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'month':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and int(elem.attrib['type']) in widths:
continue
widths[int(elem.attrib.get('type'))] = \
text_type(elem.text)
elif elem.tag == 'alias':
ctxts[width_type] = Alias(
_translate_alias(['months', ctxt_type, width_type],
elem.attrib['path'])
)
days = data.setdefault('days', {})
for ctxt in calendar.findall('days/dayContext'):
ctxt_type = ctxt.attrib['type']
ctxts = days.setdefault(ctxt_type, {})
for width in ctxt.findall('dayWidth'):
width_type = width.attrib['type']
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'day':
dtype = weekdays[elem.attrib['type']]
if ('draft' in elem.attrib or
'alt' not in elem.attrib) \
and dtype in widths:
continue
widths[dtype] = text_type(elem.text)
elif elem.tag == 'alias':
ctxts[width_type] = Alias(
_translate_alias(['days', ctxt_type, width_type],
elem.attrib['path'])
)
quarters = data.setdefault('quarters', {})
for ctxt in calendar.findall('quarters/quarterContext'):
ctxt_type = ctxt.attrib['type']
ctxts = quarters.setdefault(ctxt.attrib['type'], {})
for width in ctxt.findall('quarterWidth'):
width_type = width.attrib['type']
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'quarter':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and int(elem.attrib['type']) in widths:
continue
widths[int(elem.attrib['type'])] = text_type(elem.text)
elif elem.tag == 'alias':
ctxts[width_type] = Alias(
_translate_alias(['quarters', ctxt_type,
width_type],
elem.attrib['path']))
eras = data.setdefault('eras', {})
for width in calendar.findall('eras/*'):
width_type = NAME_MAP[width.tag]
widths = eras.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'era':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and int(elem.attrib['type']) in widths:
continue
widths[int(elem.attrib.get('type'))] = text_type(elem.text)
elif elem.tag == 'alias':
eras[width_type] = Alias(
_translate_alias(['eras', width_type],
elem.attrib['path'])
)
# AM/PM
periods = data.setdefault('periods', {})
for day_period_width in calendar.findall(
'dayPeriods/dayPeriodContext/dayPeriodWidth'):
if day_period_width.attrib['type'] == 'wide':
for day_period in day_period_width.findall('dayPeriod'):
if 'alt' not in day_period.attrib:
periods[day_period.attrib['type']] = text_type(
day_period.text)
date_formats = data.setdefault('date_formats', {})
for format in calendar.findall('dateFormats'):
for elem in format.getiterator():
if elem.tag == 'dateFormatLength':
if 'draft' in elem.attrib and \
elem.attrib.get('type') in date_formats:
continue
try:
date_formats[elem.attrib.get('type')] = \
dates.parse_pattern(text_type(
elem.findtext('dateFormat/pattern')))
except ValueError as e:
error(e)
elif elem.tag == 'alias':
date_formats = Alias(_translate_alias(
['date_formats'], elem.attrib['path'])
)
time_formats = data.setdefault('time_formats', {})
for format in calendar.findall('timeFormats'):
for elem in format.getiterator():
if elem.tag == 'timeFormatLength':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in time_formats:
continue
try:
time_formats[elem.attrib.get('type')] = \
dates.parse_pattern(text_type(
elem.findtext('timeFormat/pattern')))
except ValueError as e:
error(e)
elif elem.tag == 'alias':
time_formats = Alias(_translate_alias(
['time_formats'], elem.attrib['path'])
)
datetime_formats = data.setdefault('datetime_formats', {})
for format in calendar.findall('dateTimeFormats'):
for elem in format.getiterator():
if elem.tag == 'dateTimeFormatLength':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in datetime_formats:
continue
try:
datetime_formats[elem.attrib.get('type')] = \
text_type(elem.findtext('dateTimeFormat/pattern'))
except ValueError as e:
error(e)
elif elem.tag == 'alias':
datetime_formats = Alias(_translate_alias(
['datetime_formats'], elem.attrib['path'])
)
# <numbers>
number_symbols = data.setdefault('number_symbols', {})
for elem in tree.findall('.//numbers/symbols/*'):
if ('draft' in elem.attrib or 'alt' in elem.attrib):
continue
number_symbols[elem.tag] = text_type(elem.text)
decimal_formats = data.setdefault('decimal_formats', {})
for elem in tree.findall('.//decimalFormats/decimalFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in decimal_formats:
continue
if elem.findall('./alias'):
# TODO map the alias to its target
continue
pattern = text_type(elem.findtext('./decimalFormat/pattern'))
decimal_formats[elem.attrib.get('type')] = \
numbers.parse_pattern(pattern)
scientific_formats = data.setdefault('scientific_formats', {})
for elem in tree.findall('.//scientificFormats/scientificFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in scientific_formats:
continue
pattern = text_type(elem.findtext('scientificFormat/pattern'))
scientific_formats[elem.attrib.get('type')] = \
numbers.parse_pattern(pattern)
currency_formats = data.setdefault('currency_formats', {})
for elem in tree.findall('.//currencyFormats/currencyFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in currency_formats:
continue
pattern = text_type(elem.findtext('currencyFormat/pattern'))
currency_formats[elem.attrib.get('type')] = \
numbers.parse_pattern(pattern)
percent_formats = data.setdefault('percent_formats', {})
for elem in tree.findall('.//percentFormats/percentFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in percent_formats:
continue
pattern = text_type(elem.findtext('percentFormat/pattern'))
percent_formats[elem.attrib.get('type')] = \
numbers.parse_pattern(pattern)
currency_names = data.setdefault('currency_names', {})
currency_names_plural = data.setdefault('currency_names_plural', {})
currency_symbols = data.setdefault('currency_symbols', {})
for elem in tree.findall('.//currencies/currency'):
code = elem.attrib['type']
for name in elem.findall('displayName'):
if ('draft' in name.attrib) and code in currency_names:
continue
if 'count' in name.attrib:
currency_names_plural.setdefault(code, {})[
name.attrib['count']] = text_type(name.text)
else:
currency_names[code] = text_type(name.text)
# TODO: support choice patterns for currency symbol selection
symbol = elem.find('symbol')
if symbol is not None and 'draft' not in symbol.attrib \
and 'choice' not in symbol.attrib:
currency_symbols[code] = text_type(symbol.text)
# <units>
unit_patterns = data.setdefault('unit_patterns', {})
for elem in tree.findall('.//units/unitLength'):
unit_length_type = elem.attrib['type']
for unit in elem.findall('unit'):
unit_type = unit.attrib['type']
for pattern in unit.findall('unitPattern'):
box = unit_type
box += ':' + unit_length_type
unit_patterns.setdefault(box, {})[pattern.attrib['count']] = \
text_type(pattern.text)
date_fields = data.setdefault('date_fields', {})
for elem in tree.findall('.//dates/fields/field'):
field_type = elem.attrib['type']
date_fields.setdefault(field_type, {})
for rel_time in elem.findall('relativeTime'):
rel_time_type = rel_time.attrib['type']
for pattern in rel_time.findall('relativeTimePattern'):
date_fields[field_type].setdefault(rel_time_type, {})\
[pattern.attrib['count']] = text_type(pattern.text)
outfile = open(data_filename, 'wb')
try:
pickle.dump(data, outfile, 2)
finally:
outfile.close()
def load_i18n(canonical_host, canonical_scheme, project_root, tell_sentry):
# Load the locales
localeDir = os.path.join(project_root, 'i18n', 'core')
locales = LOCALES
source_strings = {}
for file in os.listdir(localeDir):
try:
parts = file.split(".")
if not (len(parts) == 2 and parts[1] == "po"):
continue
lang = parts[0]
with open(os.path.join(localeDir, file)) as f:
l = locales[lang.lower()] = Locale(lang)
c = l.catalog = read_po(f)
share_source_strings(c, source_strings)
c.plural_func = get_function_from_rule(c.plural_expr)
replace_unused_singulars(c)
try:
l.countries = make_sorted_dict(COUNTRIES, l.territories)
except KeyError:
l.countries = COUNTRIES
try:
l.languages_2 = make_sorted_dict(LANGUAGES_2, l.languages)
except KeyError:
l.languages_2 = LANGUAGES_2
except Exception as e:
tell_sentry(e, {})
del source_strings
# Prepare a unique and sorted list for use in the language switcher
percent = lambda l, total: sum((percent(s, len(s))
if isinstance(s, tuple) else 1) for s in l
if s) / total
for l in list(locales.values()):
if l.language == 'en':
l.completion = 1
continue
l.completion = percent(
[m.string for m in l.catalog if m.id and not m.fuzzy],
len(l.catalog))
if l.completion == 0:
del locales[l.language]
loc_url = canonical_scheme + '://%s.' + canonical_host
domain, port = (canonical_host.split(':') + [None])[:2]
port = int(port) if port else socket.getservbyname(canonical_scheme, 'tcp')
subdomains = {
k: loc_url % k
for k in locales if resolve(k + '.' + domain, port)
}
lang_list = sorted(
((l.completion, l.language, l.language_name.title(),
loc_url % l.language)
for l in set(locales.values()) if l.completion > 0.5),
key=lambda t: (-t[0], t[1]),
)
# Add year-less date format
year_re = re.compile(r'(^y+[^a-zA-Z]+|[^a-zA-Z]+y+$)')
for l in locales.values():
short_format = l.date_formats['short'].pattern
assert short_format[0] == 'y' or short_format[-1] == 'y', (
l.language, short_format)
l.date_formats['short_yearless'] = year_re.sub('', short_format)
# Add aliases
for k, v in list(locales.items()):
locales.setdefault(ALIASES.get(k, k), v)
locales.setdefault(ALIASES_R.get(k, k), v)
for k, v in list(locales.items()):
locales.setdefault(k.split('_', 1)[0], v)
# Patch the locales to look less formal
locales['fr'].currency_formats['standard'] = parse_pattern(
'#,##0.00\u202f\xa4')
locales['fr'].currency_symbols['USD'] = '$'
locales['fr'].currencies['USD'] = 'dollar états-unien'
# Load the markdown files
docs = {}
heading_re = re.compile(r'^(#+ )', re.M)
for path in find_files(os.path.join(project_root, 'i18n'), '*.md'):
d, b = os.path.split(path)
doc = os.path.basename(d)
lang = b[:-3]
with open(path, 'rb') as f:
md = f.read().decode('utf8')
if md.startswith('# '):
md = '\n'.join(md.split('\n')[1:]).strip()
md = heading_re.sub(r'##\1', md)
docs.setdefault(doc, {}).__setitem__(lang, markdown.render(md))
return {
'docs': docs,
'lang_list': lang_list,
'locales': locales,
'subdomains': subdomains
}
def main():
parser = OptionParser(usage='%prog path/to/cldr')
options, args = parser.parse_args()
if len(args) != 1:
parser.error('incorrect number of arguments')
srcdir = args[0]
destdir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), '..',
'babel')
sup_filename = os.path.join(srcdir, 'supplemental', 'supplementalData.xml')
bcp47_timezone = parse(os.path.join(srcdir, 'bcp47', 'timezone.xml'))
sup_windows_zones = parse(
os.path.join(srcdir, 'supplemental', 'windowsZones.xml'))
sup_metadata = parse(
os.path.join(srcdir, 'supplemental', 'supplementalMetadata.xml'))
sup_likely = parse(
os.path.join(srcdir, 'supplemental', 'likelySubtags.xml'))
sup = parse(sup_filename)
# Import global data from the supplemental files
global_path = os.path.join(destdir, 'global.dat')
global_data = {}
if need_conversion(global_path, global_data, sup_filename):
territory_zones = global_data.setdefault('territory_zones', {})
zone_aliases = global_data.setdefault('zone_aliases', {})
zone_territories = global_data.setdefault('zone_territories', {})
win_mapping = global_data.setdefault('windows_zone_mapping', {})
language_aliases = global_data.setdefault('language_aliases', {})
territory_aliases = global_data.setdefault('territory_aliases', {})
script_aliases = global_data.setdefault('script_aliases', {})
variant_aliases = global_data.setdefault('variant_aliases', {})
likely_subtags = global_data.setdefault('likely_subtags', {})
territory_currencies = global_data.setdefault('territory_currencies',
{})
# create auxiliary zone->territory map from the windows zones (we don't set
# the 'zones_territories' map directly here, because there are some zones
# aliases listed and we defer the decision of which ones to choose to the
# 'bcp47' data
_zone_territory_map = {}
for map_zone in sup_windows_zones.findall(
'.//windowsZones/mapTimezones/mapZone'):
if map_zone.attrib.get('territory') == '001':
win_mapping[map_zone.attrib['other']] = \
map_zone.attrib['type'].split()[0]
for tzid in text_type(map_zone.attrib['type']).split():
_zone_territory_map[tzid] = \
text_type(map_zone.attrib['territory'])
for key_elem in bcp47_timezone.findall('.//keyword/key'):
if key_elem.attrib['name'] == 'tz':
for elem in key_elem.findall('type'):
aliases = text_type(elem.attrib['alias']).split()
tzid = aliases.pop(0)
territory = _zone_territory_map.get(tzid, '001')
territory_zones.setdefault(territory, []).append(tzid)
zone_territories[tzid] = territory
for alias in aliases:
zone_aliases[alias] = tzid
break
# Import Metazone mapping
meta_zones = global_data.setdefault('meta_zones', {})
tzsup = parse(os.path.join(srcdir, 'supplemental', 'metaZones.xml'))
for elem in tzsup.findall('.//timezone'):
for child in elem.findall('usesMetazone'):
if 'to' not in child.attrib: # FIXME: support old mappings
meta_zones[elem.attrib['type']] = child.attrib['mzone']
# Language aliases
for alias in sup_metadata.findall('.//alias/languageAlias'):
# We don't have a use for those at the moment. They don't
# pass our parser anyways.
if '-' in alias.attrib['type']:
continue
language_aliases[
alias.attrib['type']] = alias.attrib['replacement']
# Territory aliases
for alias in sup_metadata.findall('.//alias/territoryAlias'):
territory_aliases[alias.attrib['type']] = \
alias.attrib['replacement'].split()
# Script aliases
for alias in sup_metadata.findall('.//alias/scriptAlias'):
script_aliases[alias.attrib['type']] = alias.attrib['replacement']
# Variant aliases
for alias in sup_metadata.findall('.//alias/variantAlias'):
repl = alias.attrib.get('replacement')
if repl:
variant_aliases[alias.attrib['type']] = repl
# Likely subtags
for likely_subtag in sup_likely.findall(
'.//likelySubtags/likelySubtag'):
likely_subtags[likely_subtag.attrib['from']] = \
likely_subtag.attrib['to']
# Currencies in territories
for region in sup.findall('.//currencyData/region'):
region_code = region.attrib['iso3166']
region_currencies = []
for currency in region.findall('./currency'):
cur_start = _parse_currency_date(currency.attrib.get('from'))
cur_end = _parse_currency_date(currency.attrib.get('to'))
region_currencies.append(
(currency.attrib['iso4217'], cur_start, cur_end,
currency.attrib.get('tender', 'true') == 'true'))
region_currencies.sort(key=_currency_sort_key)
territory_currencies[region_code] = region_currencies
outfile = open(global_path, 'wb')
try:
pickle.dump(global_data, outfile, 2)
finally:
outfile.close()
# build a territory containment mapping for inheritance
regions = {}
for elem in sup.findall('.//territoryContainment/group'):
regions[elem.attrib['type']] = elem.attrib['contains'].split()
# Resolve territory containment
territory_containment = {}
region_items = sorted(regions.items())
for group, territory_list in region_items:
for territory in territory_list:
containers = territory_containment.setdefault(territory, set([]))
if group in territory_containment:
containers |= territory_containment[group]
containers.add(group)
# prepare the per-locale plural rules definitions
plural_rules = {}
prsup = parse(os.path.join(srcdir, 'supplemental', 'plurals.xml'))
for elem in prsup.findall('.//plurals/pluralRules'):
rules = []
for rule in elem.findall('pluralRule'):
rules.append((rule.attrib['count'], text_type(rule.text)))
pr = PluralRule(rules)
for locale in elem.attrib['locales'].split():
plural_rules[locale] = pr
filenames = os.listdir(os.path.join(srcdir, 'main'))
filenames.remove('root.xml')
filenames.sort(key=len)
filenames.insert(0, 'root.xml')
for filename in filenames:
stem, ext = os.path.splitext(filename)
if ext != '.xml':
continue
full_filename = os.path.join(srcdir, 'main', filename)
data_filename = os.path.join(destdir, 'localedata', stem + '.dat')
data = {}
if not need_conversion(data_filename, data, full_filename):
continue
tree = parse(full_filename)
language = None
elem = tree.find('.//identity/language')
if elem is not None:
language = elem.attrib['type']
territory = None
elem = tree.find('.//identity/territory')
if elem is not None:
territory = elem.attrib['type']
else:
territory = '001' # world
regions = territory_containment.get(territory, [])
log('Processing %s (Language = %s; Territory = %s)', filename,
language, territory)
# plural rules
locale_id = '_'.join(
filter(None, [language, territory != '001' and territory or None]))
if locale_id in plural_rules:
data['plural_form'] = plural_rules[locale_id]
# <localeDisplayNames>
territories = data.setdefault('territories', {})
for elem in tree.findall('.//territories/territory'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in territories:
continue
territories[elem.attrib['type']] = _text(elem)
languages = data.setdefault('languages', {})
for elem in tree.findall('.//languages/language'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in languages:
continue
languages[elem.attrib['type']] = _text(elem)
variants = data.setdefault('variants', {})
for elem in tree.findall('.//variants/variant'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in variants:
continue
variants[elem.attrib['type']] = _text(elem)
scripts = data.setdefault('scripts', {})
for elem in tree.findall('.//scripts/script'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib['type'] in scripts:
continue
scripts[elem.attrib['type']] = _text(elem)
# <dates>
week_data = data.setdefault('week_data', {})
supelem = sup.find('.//weekData')
for elem in supelem.findall('minDays'):
territories = elem.attrib['territories'].split()
if territory in territories or any(
[r in territories for r in regions]):
week_data['min_days'] = int(elem.attrib['count'])
for elem in supelem.findall('firstDay'):
territories = elem.attrib['territories'].split()
if territory in territories or any(
[r in territories for r in regions]):
week_data['first_day'] = weekdays[elem.attrib['day']]
for elem in supelem.findall('weekendStart'):
territories = elem.attrib['territories'].split()
if territory in territories or any(
[r in territories for r in regions]):
week_data['weekend_start'] = weekdays[elem.attrib['day']]
for elem in supelem.findall('weekendEnd'):
territories = elem.attrib['territories'].split()
if territory in territories or any(
[r in territories for r in regions]):
week_data['weekend_end'] = weekdays[elem.attrib['day']]
zone_formats = data.setdefault('zone_formats', {})
for elem in tree.findall('.//timeZoneNames/gmtFormat'):
if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
zone_formats['gmt'] = text_type(elem.text).replace('{0}', '%s')
break
for elem in tree.findall('.//timeZoneNames/regionFormat'):
if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
zone_formats['region'] = text_type(elem.text).replace(
'{0}', '%s')
break
for elem in tree.findall('.//timeZoneNames/fallbackFormat'):
if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
zone_formats['fallback'] = text_type(elem.text) \
.replace('{0}', '%(0)s').replace('{1}', '%(1)s')
break
for elem in tree.findall('.//timeZoneNames/fallbackRegionFormat'):
if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
zone_formats['fallback_region'] = text_type(elem.text) \
.replace('{0}', '%(0)s').replace('{1}', '%(1)s')
break
time_zones = data.setdefault('time_zones', {})
for elem in tree.findall('.//timeZoneNames/zone'):
info = {}
city = elem.findtext('exemplarCity')
if city:
info['city'] = text_type(city)
for child in elem.findall('long/*'):
info.setdefault('long', {})[child.tag] = text_type(child.text)
for child in elem.findall('short/*'):
info.setdefault('short', {})[child.tag] = text_type(child.text)
time_zones[elem.attrib['type']] = info
meta_zones = data.setdefault('meta_zones', {})
for elem in tree.findall('.//timeZoneNames/metazone'):
info = {}
city = elem.findtext('exemplarCity')
if city:
info['city'] = text_type(city)
for child in elem.findall('long/*'):
info.setdefault('long', {})[child.tag] = text_type(child.text)
for child in elem.findall('short/*'):
info.setdefault('short', {})[child.tag] = text_type(child.text)
meta_zones[elem.attrib['type']] = info
for calendar in tree.findall('.//calendars/calendar'):
if calendar.attrib['type'] != 'gregorian':
# TODO: support other calendar types
continue
months = data.setdefault('months', {})
for ctxt in calendar.findall('months/monthContext'):
ctxt_type = ctxt.attrib['type']
ctxts = months.setdefault(ctxt_type, {})
for width in ctxt.findall('monthWidth'):
width_type = width.attrib['type']
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'month':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and int(elem.attrib['type']) in widths:
continue
widths[int(elem.attrib.get('type'))] = \
text_type(elem.text)
elif elem.tag == 'alias':
ctxts[width_type] = Alias(
_translate_alias(
['months', ctxt_type, width_type],
elem.attrib['path']))
days = data.setdefault('days', {})
for ctxt in calendar.findall('days/dayContext'):
ctxt_type = ctxt.attrib['type']
ctxts = days.setdefault(ctxt_type, {})
for width in ctxt.findall('dayWidth'):
width_type = width.attrib['type']
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'day':
dtype = weekdays[elem.attrib['type']]
if ('draft' in elem.attrib or
'alt' not in elem.attrib) \
and dtype in widths:
continue
widths[dtype] = text_type(elem.text)
elif elem.tag == 'alias':
ctxts[width_type] = Alias(
_translate_alias(
['days', ctxt_type, width_type],
elem.attrib['path']))
quarters = data.setdefault('quarters', {})
for ctxt in calendar.findall('quarters/quarterContext'):
ctxt_type = ctxt.attrib['type']
ctxts = quarters.setdefault(ctxt.attrib['type'], {})
for width in ctxt.findall('quarterWidth'):
width_type = width.attrib['type']
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'quarter':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and int(elem.attrib['type']) in widths:
continue
widths[int(elem.attrib['type'])] = text_type(
elem.text)
elif elem.tag == 'alias':
ctxts[width_type] = Alias(
_translate_alias(
['quarters', ctxt_type, width_type],
elem.attrib['path']))
eras = data.setdefault('eras', {})
for width in calendar.findall('eras/*'):
width_type = NAME_MAP[width.tag]
widths = eras.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == 'era':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and int(elem.attrib['type']) in widths:
continue
widths[int(elem.attrib.get('type'))] = text_type(
elem.text)
elif elem.tag == 'alias':
eras[width_type] = Alias(
_translate_alias(['eras', width_type],
elem.attrib['path']))
# AM/PM
periods = data.setdefault('periods', {})
for day_period_width in calendar.findall(
'dayPeriods/dayPeriodContext/dayPeriodWidth'):
if day_period_width.attrib['type'] == 'wide':
for day_period in day_period_width.findall('dayPeriod'):
if 'alt' not in day_period.attrib:
periods[day_period.attrib['type']] = text_type(
day_period.text)
date_formats = data.setdefault('date_formats', {})
for format in calendar.findall('dateFormats'):
for elem in format.getiterator():
if elem.tag == 'dateFormatLength':
if 'draft' in elem.attrib and \
elem.attrib.get('type') in date_formats:
continue
try:
date_formats[elem.attrib.get('type')] = \
dates.parse_pattern(text_type(
elem.findtext('dateFormat/pattern')))
except ValueError as e:
error(e)
elif elem.tag == 'alias':
date_formats = Alias(
_translate_alias(['date_formats'],
elem.attrib['path']))
time_formats = data.setdefault('time_formats', {})
for format in calendar.findall('timeFormats'):
for elem in format.getiterator():
if elem.tag == 'timeFormatLength':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in time_formats:
continue
try:
time_formats[elem.attrib.get('type')] = \
dates.parse_pattern(text_type(
elem.findtext('timeFormat/pattern')))
except ValueError as e:
error(e)
elif elem.tag == 'alias':
time_formats = Alias(
_translate_alias(['time_formats'],
elem.attrib['path']))
datetime_formats = data.setdefault('datetime_formats', {})
for format in calendar.findall('dateTimeFormats'):
for elem in format.getiterator():
if elem.tag == 'dateTimeFormatLength':
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in datetime_formats:
continue
try:
datetime_formats[elem.attrib.get('type')] = \
text_type(elem.findtext('dateTimeFormat/pattern'))
except ValueError as e:
error(e)
elif elem.tag == 'alias':
datetime_formats = Alias(
_translate_alias(['datetime_formats'],
elem.attrib['path']))
# <numbers>
number_symbols = data.setdefault('number_symbols', {})
for elem in tree.findall('.//numbers/symbols/*'):
if ('draft' in elem.attrib or 'alt' in elem.attrib):
continue
number_symbols[elem.tag] = text_type(elem.text)
decimal_formats = data.setdefault('decimal_formats', {})
for elem in tree.findall('.//decimalFormats/decimalFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in decimal_formats:
continue
if elem.findall('./alias'):
# TODO map the alias to its target
continue
pattern = text_type(elem.findtext('./decimalFormat/pattern'))
decimal_formats[elem.attrib.get('type')] = \
numbers.parse_pattern(pattern)
scientific_formats = data.setdefault('scientific_formats', {})
for elem in tree.findall(
'.//scientificFormats/scientificFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in scientific_formats:
continue
pattern = text_type(elem.findtext('scientificFormat/pattern'))
scientific_formats[elem.attrib.get('type')] = \
numbers.parse_pattern(pattern)
currency_formats = data.setdefault('currency_formats', {})
for elem in tree.findall('.//currencyFormats/currencyFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in currency_formats:
continue
pattern = text_type(elem.findtext('currencyFormat/pattern'))
currency_formats[elem.attrib.get('type')] = \
numbers.parse_pattern(pattern)
percent_formats = data.setdefault('percent_formats', {})
for elem in tree.findall('.//percentFormats/percentFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in percent_formats:
continue
pattern = text_type(elem.findtext('percentFormat/pattern'))
percent_formats[elem.attrib.get('type')] = \
numbers.parse_pattern(pattern)
currency_names = data.setdefault('currency_names', {})
currency_names_plural = data.setdefault('currency_names_plural', {})
currency_symbols = data.setdefault('currency_symbols', {})
for elem in tree.findall('.//currencies/currency'):
code = elem.attrib['type']
for name in elem.findall('displayName'):
if ('draft' in name.attrib) and code in currency_names:
continue
if 'count' in name.attrib:
currency_names_plural.setdefault(
code, {})[name.attrib['count']] = text_type(name.text)
else:
currency_names[code] = text_type(name.text)
# TODO: support choice patterns for currency symbol selection
symbol = elem.find('symbol')
if symbol is not None and 'draft' not in symbol.attrib \
and 'choice' not in symbol.attrib:
currency_symbols[code] = text_type(symbol.text)
# <units>
unit_patterns = data.setdefault('unit_patterns', {})
for elem in tree.findall('.//units/unit'):
unit_type = elem.attrib['type']
for pattern in elem.findall('unitPattern'):
box = unit_type
if 'alt' in pattern.attrib:
box += ':' + pattern.attrib['alt']
unit_patterns.setdefault(box, {})[pattern.attrib['count']] = \
text_type(pattern.text)
outfile = open(data_filename, 'wb')
try:
pickle.dump(data, outfile, 2)
finally:
outfile.close()
def load_i18n(canonical_host, canonical_scheme, project_root, tell_sentry):
def compute_percentage(it, total):
return sum(
(compute_percentage(s, len(s)) if isinstance(s, tuple) else 1)
for s in it if s) / total
# Load the base locales
localeDir = os.path.join(project_root, 'i18n', 'core')
locales = LOCALES
source_strings = {}
for file in os.listdir(localeDir):
try:
parts = file.split(".")
if not (len(parts) == 2 and parts[1] == "po"):
continue
lang = parts[0]
with open(os.path.join(localeDir, file), 'rb') as f:
l = Locale(lang)
c = l.catalog = read_po(f)
share_source_strings(c, source_strings)
c.plural_func = get_function_from_rule(c.plural_expr)
replace_unused_singulars(c)
l.completion = compute_percentage(
(m.string for m in c if m.id and not m.fuzzy), len(c))
if l.completion == 0:
continue
else:
locales[lang.lower()] = l
try:
l.countries = make_sorted_dict(COUNTRIES, l.territories)
except KeyError:
l.countries = COUNTRIES
try:
l.languages_2 = make_sorted_dict(LANGUAGES_2, l.languages)
except KeyError:
l.languages_2 = LANGUAGES_2
except Exception as e:
tell_sentry(e)
del source_strings
# Load the variants
for loc_id in babel.localedata.locale_identifiers():
if loc_id in locales:
continue
i = loc_id.rfind('_')
if i == -1:
continue
base = locales.get(loc_id[:i])
if base:
l = locales[loc_id.lower()] = Locale.parse(loc_id)
l.catalog = base.catalog
l.completion = base.completion
l.countries = base.countries
l.languages_2 = base.languages_2
# Unload the Babel data that we no longer need
# We load a lot of data to populate the LANGUAGE_NAMES dict, we don't want
# to keep it all in RAM.
used_data_dict_addresses = set(id(l._data._data) for l in locales.values())
for key, data_dict in list(babel.localedata._cache.items()):
if id(data_dict) not in used_data_dict_addresses:
del babel.localedata._cache[key]
# Prepare a unique and sorted list for use in the language switcher
loc_url = canonical_scheme + '://%s.' + canonical_host
domain, port = (canonical_host.split(':') + [None])[:2]
port = int(port) if port else socket.getservbyname(canonical_scheme, 'tcp')
subdomains = {
l.subdomain: loc_url % l.subdomain
for l in locales.values()
if not l.territory and resolve(l.subdomain + '.' + domain, port)
}
lang_list = sorted(
((l.completion, l.language, l.language_name.title(),
loc_url % l.subdomain) for l in set(locales.values())
if not l.territory and l.completion > 0.5),
key=lambda t: (-t[0], t[1]),
)
# Add year-less date format
year_re = re.compile(r'(^y+[^a-zA-Z]+|[^a-zA-Z]+y+$)')
for l in locales.values():
short_format = l.date_formats['short'].pattern
assert short_format[0] == 'y' or short_format[-1] == 'y', (
l.language, short_format)
l.date_formats['short_yearless'] = year_re.sub('', short_format)
# Add aliases
for k, v in list(locales.items()):
locales.setdefault(ALIASES.get(k, k), v)
locales.setdefault(ALIASES_R.get(k, k), v)
for k, v in list(locales.items()):
locales.setdefault(k.split('_', 1)[0], v)
# Add universal strings
# These strings don't need to be translated, but they have to be in the catalogs
# so that they're counted as translated.
for l in locales.values():
l.catalog.add("PayPal", "PayPal")
# Patch the locales to look less formal
locales['fr'].currency_formats['standard'] = parse_pattern(
'#,##0.00\u202f\xa4')
locales['fr'].currencies['USD'] = 'dollar états-unien'
# Load the markdown files
docs = {}
heading_re = re.compile(r'^(#+ )', re.M)
for path in find_files(os.path.join(project_root, 'i18n'), '*.md'):
d, b = os.path.split(path)
doc = os.path.basename(d)
lang = b[:-3]
with open(path, 'rb') as f:
md = f.read().decode('utf8')
if md.startswith('# '):
md = '\n'.join(md.split('\n')[1:]).strip()
md = heading_re.sub(r'##\1', md)
docs.setdefault(doc, {}).__setitem__(lang, markdown.render(md))
return {
'docs': docs,
'lang_list': lang_list,
'locales': locales,
'subdomains': subdomains
}
def main():
parser = OptionParser(usage="%prog path/to/cldr")
options, args = parser.parse_args()
if len(args) != 1:
parser.error("incorrect number of arguments")
srcdir = args[0]
destdir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "..", "babel")
sup_filename = os.path.join(srcdir, "supplemental", "supplementalData.xml")
bcp47_timezone = parse(os.path.join(srcdir, "bcp47", "timezone.xml"))
sup_windows_zones = parse(os.path.join(srcdir, "supplemental", "windowsZones.xml"))
sup_metadata = parse(os.path.join(srcdir, "supplemental", "supplementalMetadata.xml"))
sup_likely = parse(os.path.join(srcdir, "supplemental", "likelySubtags.xml"))
sup = parse(sup_filename)
# Import global data from the supplemental files
global_path = os.path.join(destdir, "global.dat")
global_data = {}
if need_conversion(global_path, global_data, sup_filename):
territory_zones = global_data.setdefault("territory_zones", {})
zone_aliases = global_data.setdefault("zone_aliases", {})
zone_territories = global_data.setdefault("zone_territories", {})
win_mapping = global_data.setdefault("windows_zone_mapping", {})
language_aliases = global_data.setdefault("language_aliases", {})
territory_aliases = global_data.setdefault("territory_aliases", {})
script_aliases = global_data.setdefault("script_aliases", {})
variant_aliases = global_data.setdefault("variant_aliases", {})
likely_subtags = global_data.setdefault("likely_subtags", {})
territory_currencies = global_data.setdefault("territory_currencies", {})
parent_exceptions = global_data.setdefault("parent_exceptions", {})
currency_fractions = global_data.setdefault("currency_fractions", {})
# create auxiliary zone->territory map from the windows zones (we don't set
# the 'zones_territories' map directly here, because there are some zones
# aliases listed and we defer the decision of which ones to choose to the
# 'bcp47' data
_zone_territory_map = {}
for map_zone in sup_windows_zones.findall(".//windowsZones/mapTimezones/mapZone"):
if map_zone.attrib.get("territory") == "001":
win_mapping[map_zone.attrib["other"]] = map_zone.attrib["type"].split()[0]
for tzid in text_type(map_zone.attrib["type"]).split():
_zone_territory_map[tzid] = text_type(map_zone.attrib["territory"])
for key_elem in bcp47_timezone.findall(".//keyword/key"):
if key_elem.attrib["name"] == "tz":
for elem in key_elem.findall("type"):
if "deprecated" not in elem.attrib:
aliases = text_type(elem.attrib["alias"]).split()
tzid = aliases.pop(0)
territory = _zone_territory_map.get(tzid, "001")
territory_zones.setdefault(territory, []).append(tzid)
zone_territories[tzid] = territory
for alias in aliases:
zone_aliases[alias] = tzid
break
# Import Metazone mapping
meta_zones = global_data.setdefault("meta_zones", {})
tzsup = parse(os.path.join(srcdir, "supplemental", "metaZones.xml"))
for elem in tzsup.findall(".//timezone"):
for child in elem.findall("usesMetazone"):
if "to" not in child.attrib: # FIXME: support old mappings
meta_zones[elem.attrib["type"]] = child.attrib["mzone"]
# Language aliases
for alias in sup_metadata.findall(".//alias/languageAlias"):
# We don't have a use for those at the moment. They don't
# pass our parser anyways.
if "_" in alias.attrib["type"]:
continue
language_aliases[alias.attrib["type"]] = alias.attrib["replacement"]
# Territory aliases
for alias in sup_metadata.findall(".//alias/territoryAlias"):
territory_aliases[alias.attrib["type"]] = alias.attrib["replacement"].split()
# Script aliases
for alias in sup_metadata.findall(".//alias/scriptAlias"):
script_aliases[alias.attrib["type"]] = alias.attrib["replacement"]
# Variant aliases
for alias in sup_metadata.findall(".//alias/variantAlias"):
repl = alias.attrib.get("replacement")
if repl:
variant_aliases[alias.attrib["type"]] = repl
# Likely subtags
for likely_subtag in sup_likely.findall(".//likelySubtags/likelySubtag"):
likely_subtags[likely_subtag.attrib["from"]] = likely_subtag.attrib["to"]
# Currencies in territories
for region in sup.findall(".//currencyData/region"):
region_code = region.attrib["iso3166"]
region_currencies = []
for currency in region.findall("./currency"):
cur_start = _parse_currency_date(currency.attrib.get("from"))
cur_end = _parse_currency_date(currency.attrib.get("to"))
region_currencies.append(
(currency.attrib["iso4217"], cur_start, cur_end, currency.attrib.get("tender", "true") == "true")
)
region_currencies.sort(key=_currency_sort_key)
territory_currencies[region_code] = region_currencies
# Explicit parent locales
for paternity in sup.findall(".//parentLocales/parentLocale"):
parent = paternity.attrib["parent"]
for child in paternity.attrib["locales"].split():
parent_exceptions[child] = parent
# Currency decimal and rounding digits
for fraction in sup.findall(".//currencyData/fractions/info"):
cur_code = fraction.attrib["iso4217"]
cur_digits = int(fraction.attrib["digits"])
cur_rounding = int(fraction.attrib["rounding"])
cur_cdigits = int(fraction.attrib.get("cashDigits", cur_digits))
cur_crounding = int(fraction.attrib.get("cashRounding", cur_rounding))
currency_fractions[cur_code] = (cur_digits, cur_rounding, cur_cdigits, cur_crounding)
outfile = open(global_path, "wb")
try:
pickle.dump(global_data, outfile, 2)
finally:
outfile.close()
# build a territory containment mapping for inheritance
regions = {}
for elem in sup.findall(".//territoryContainment/group"):
regions[elem.attrib["type"]] = elem.attrib["contains"].split()
# Resolve territory containment
territory_containment = {}
region_items = sorted(regions.items())
for group, territory_list in region_items:
for territory in territory_list:
containers = territory_containment.setdefault(territory, set([]))
if group in territory_containment:
containers |= territory_containment[group]
containers.add(group)
# prepare the per-locale plural rules definitions
plural_rules = {}
prsup = parse(os.path.join(srcdir, "supplemental", "plurals.xml"))
for elem in prsup.findall(".//plurals/pluralRules"):
rules = []
for rule in elem.findall("pluralRule"):
rules.append((rule.attrib["count"], text_type(rule.text)))
pr = PluralRule(rules)
for locale in elem.attrib["locales"].split():
plural_rules[locale] = pr
filenames = os.listdir(os.path.join(srcdir, "main"))
filenames.remove("root.xml")
filenames.sort(key=len)
filenames.insert(0, "root.xml")
for filename in filenames:
stem, ext = os.path.splitext(filename)
if ext != ".xml":
continue
full_filename = os.path.join(srcdir, "main", filename)
data_filename = os.path.join(destdir, "locale-data", stem + ".dat")
data = {}
if not need_conversion(data_filename, data, full_filename):
continue
tree = parse(full_filename)
language = None
elem = tree.find(".//identity/language")
if elem is not None:
language = elem.attrib["type"]
territory = None
elem = tree.find(".//identity/territory")
if elem is not None:
territory = elem.attrib["type"]
else:
territory = "001" # world
regions = territory_containment.get(territory, [])
log("Processing %s (Language = %s; Territory = %s)", filename, language, territory)
# plural rules
locale_id = "_".join(filter(None, [language, territory != "001" and territory or None]))
if locale_id in plural_rules:
data["plural_form"] = plural_rules[locale_id]
# <localeDisplayNames>
territories = data.setdefault("territories", {})
for elem in tree.findall(".//territories/territory"):
if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib["type"] in territories:
continue
territories[elem.attrib["type"]] = _text(elem)
languages = data.setdefault("languages", {})
for elem in tree.findall(".//languages/language"):
if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib["type"] in languages:
continue
languages[elem.attrib["type"]] = _text(elem)
variants = data.setdefault("variants", {})
for elem in tree.findall(".//variants/variant"):
if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib["type"] in variants:
continue
variants[elem.attrib["type"]] = _text(elem)
scripts = data.setdefault("scripts", {})
for elem in tree.findall(".//scripts/script"):
if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib["type"] in scripts:
continue
scripts[elem.attrib["type"]] = _text(elem)
# <dates>
week_data = data.setdefault("week_data", {})
supelem = sup.find(".//weekData")
for elem in supelem.findall("minDays"):
territories = elem.attrib["territories"].split()
if territory in territories or any([r in territories for r in regions]):
week_data["min_days"] = int(elem.attrib["count"])
for elem in supelem.findall("firstDay"):
territories = elem.attrib["territories"].split()
if territory in territories or any([r in territories for r in regions]):
week_data["first_day"] = weekdays[elem.attrib["day"]]
for elem in supelem.findall("weekendStart"):
territories = elem.attrib["territories"].split()
if territory in territories or any([r in territories for r in regions]):
week_data["weekend_start"] = weekdays[elem.attrib["day"]]
for elem in supelem.findall("weekendEnd"):
territories = elem.attrib["territories"].split()
if territory in territories or any([r in territories for r in regions]):
week_data["weekend_end"] = weekdays[elem.attrib["day"]]
zone_formats = data.setdefault("zone_formats", {})
for elem in tree.findall(".//timeZoneNames/gmtFormat"):
if "draft" not in elem.attrib and "alt" not in elem.attrib:
zone_formats["gmt"] = text_type(elem.text).replace("{0}", "%s")
break
for elem in tree.findall(".//timeZoneNames/regionFormat"):
if "draft" not in elem.attrib and "alt" not in elem.attrib:
zone_formats["region"] = text_type(elem.text).replace("{0}", "%s")
break
for elem in tree.findall(".//timeZoneNames/fallbackFormat"):
if "draft" not in elem.attrib and "alt" not in elem.attrib:
zone_formats["fallback"] = text_type(elem.text).replace("{0}", "%(0)s").replace("{1}", "%(1)s")
break
for elem in tree.findall(".//timeZoneNames/fallbackRegionFormat"):
if "draft" not in elem.attrib and "alt" not in elem.attrib:
zone_formats["fallback_region"] = text_type(elem.text).replace("{0}", "%(0)s").replace("{1}", "%(1)s")
break
time_zones = data.setdefault("time_zones", {})
for elem in tree.findall(".//timeZoneNames/zone"):
info = {}
city = elem.findtext("exemplarCity")
if city:
info["city"] = text_type(city)
for child in elem.findall("long/*"):
info.setdefault("long", {})[child.tag] = text_type(child.text)
for child in elem.findall("short/*"):
info.setdefault("short", {})[child.tag] = text_type(child.text)
time_zones[elem.attrib["type"]] = info
meta_zones = data.setdefault("meta_zones", {})
for elem in tree.findall(".//timeZoneNames/metazone"):
info = {}
city = elem.findtext("exemplarCity")
if city:
info["city"] = text_type(city)
for child in elem.findall("long/*"):
info.setdefault("long", {})[child.tag] = text_type(child.text)
for child in elem.findall("short/*"):
info.setdefault("short", {})[child.tag] = text_type(child.text)
meta_zones[elem.attrib["type"]] = info
for calendar in tree.findall(".//calendars/calendar"):
if calendar.attrib["type"] != "gregorian":
# TODO: support other calendar types
continue
months = data.setdefault("months", {})
for ctxt in calendar.findall("months/monthContext"):
ctxt_type = ctxt.attrib["type"]
ctxts = months.setdefault(ctxt_type, {})
for width in ctxt.findall("monthWidth"):
width_type = width.attrib["type"]
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == "month":
if ("draft" in elem.attrib or "alt" in elem.attrib) and int(elem.attrib["type"]) in widths:
continue
widths[int(elem.attrib.get("type"))] = text_type(elem.text)
elif elem.tag == "alias":
ctxts[width_type] = Alias(
_translate_alias(["months", ctxt_type, width_type], elem.attrib["path"])
)
days = data.setdefault("days", {})
for ctxt in calendar.findall("days/dayContext"):
ctxt_type = ctxt.attrib["type"]
ctxts = days.setdefault(ctxt_type, {})
for width in ctxt.findall("dayWidth"):
width_type = width.attrib["type"]
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == "day":
dtype = weekdays[elem.attrib["type"]]
if ("draft" in elem.attrib or "alt" not in elem.attrib) and dtype in widths:
continue
widths[dtype] = text_type(elem.text)
elif elem.tag == "alias":
ctxts[width_type] = Alias(
_translate_alias(["days", ctxt_type, width_type], elem.attrib["path"])
)
quarters = data.setdefault("quarters", {})
for ctxt in calendar.findall("quarters/quarterContext"):
ctxt_type = ctxt.attrib["type"]
ctxts = quarters.setdefault(ctxt.attrib["type"], {})
for width in ctxt.findall("quarterWidth"):
width_type = width.attrib["type"]
widths = ctxts.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == "quarter":
if ("draft" in elem.attrib or "alt" in elem.attrib) and int(elem.attrib["type"]) in widths:
continue
widths[int(elem.attrib["type"])] = text_type(elem.text)
elif elem.tag == "alias":
ctxts[width_type] = Alias(
_translate_alias(["quarters", ctxt_type, width_type], elem.attrib["path"])
)
eras = data.setdefault("eras", {})
for width in calendar.findall("eras/*"):
width_type = NAME_MAP[width.tag]
widths = eras.setdefault(width_type, {})
for elem in width.getiterator():
if elem.tag == "era":
if ("draft" in elem.attrib or "alt" in elem.attrib) and int(elem.attrib["type"]) in widths:
continue
widths[int(elem.attrib.get("type"))] = text_type(elem.text)
elif elem.tag == "alias":
eras[width_type] = Alias(_translate_alias(["eras", width_type], elem.attrib["path"]))
# AM/PM
periods = data.setdefault("periods", {})
for day_period_width in calendar.findall("dayPeriods/dayPeriodContext/dayPeriodWidth"):
if day_period_width.attrib["type"] == "wide":
for day_period in day_period_width.findall("dayPeriod"):
if "alt" not in day_period.attrib:
periods[day_period.attrib["type"]] = text_type(day_period.text)
date_formats = data.setdefault("date_formats", {})
for format in calendar.findall("dateFormats"):
for elem in format.getiterator():
if elem.tag == "dateFormatLength":
if "draft" in elem.attrib and elem.attrib.get("type") in date_formats:
continue
try:
date_formats[elem.attrib.get("type")] = dates.parse_pattern(
text_type(elem.findtext("dateFormat/pattern"))
)
except ValueError as e:
error(e)
elif elem.tag == "alias":
date_formats = Alias(_translate_alias(["date_formats"], elem.attrib["path"]))
time_formats = data.setdefault("time_formats", {})
for format in calendar.findall("timeFormats"):
for elem in format.getiterator():
if elem.tag == "timeFormatLength":
if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib.get("type") in time_formats:
continue
try:
time_formats[elem.attrib.get("type")] = dates.parse_pattern(
text_type(elem.findtext("timeFormat/pattern"))
)
except ValueError as e:
error(e)
elif elem.tag == "alias":
time_formats = Alias(_translate_alias(["time_formats"], elem.attrib["path"]))
datetime_formats = data.setdefault("datetime_formats", {})
for format in calendar.findall("dateTimeFormats"):
for elem in format.getiterator():
if elem.tag == "dateTimeFormatLength":
if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib.get(
"type"
) in datetime_formats:
continue
try:
datetime_formats[elem.attrib.get("type")] = text_type(
elem.findtext("dateTimeFormat/pattern")
)
except ValueError as e:
error(e)
elif elem.tag == "alias":
datetime_formats = Alias(_translate_alias(["datetime_formats"], elem.attrib["path"]))
# <numbers>
number_symbols = data.setdefault("number_symbols", {})
for elem in tree.findall(".//numbers/symbols/*"):
if "draft" in elem.attrib or "alt" in elem.attrib:
continue
number_symbols[elem.tag] = text_type(elem.text)
decimal_formats = data.setdefault("decimal_formats", {})
for elem in tree.findall(".//decimalFormats/decimalFormatLength"):
if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib.get("type") in decimal_formats:
continue
if elem.findall("./alias"):
# TODO map the alias to its target
continue
pattern = text_type(elem.findtext("./decimalFormat/pattern"))
decimal_formats[elem.attrib.get("type")] = numbers.parse_pattern(pattern)
scientific_formats = data.setdefault("scientific_formats", {})
for elem in tree.findall(".//scientificFormats/scientificFormatLength"):
if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib.get("type") in scientific_formats:
continue
pattern = text_type(elem.findtext("scientificFormat/pattern"))
scientific_formats[elem.attrib.get("type")] = numbers.parse_pattern(pattern)
currency_formats = data.setdefault("currency_formats", {})
for elem in tree.findall(".//currencyFormats/currencyFormatLength/currencyFormat"):
if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib.get("type") in currency_formats:
continue
for child in elem.getiterator():
if child.tag == "alias":
currency_formats[elem.attrib.get("type")] = Alias(
_translate_alias(["currency_formats", elem.attrib["type"]], child.attrib["path"])
)
elif child.tag == "pattern":
pattern = text_type(child.text)
currency_formats[elem.attrib.get("type")] = numbers.parse_pattern(pattern)
percent_formats = data.setdefault("percent_formats", {})
for elem in tree.findall(".//percentFormats/percentFormatLength"):
if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib.get("type") in percent_formats:
continue
pattern = text_type(elem.findtext("percentFormat/pattern"))
percent_formats[elem.attrib.get("type")] = numbers.parse_pattern(pattern)
currency_names = data.setdefault("currency_names", {})
currency_names_plural = data.setdefault("currency_names_plural", {})
currency_symbols = data.setdefault("currency_symbols", {})
for elem in tree.findall(".//currencies/currency"):
code = elem.attrib["type"]
for name in elem.findall("displayName"):
if ("draft" in name.attrib) and code in currency_names:
continue
if "count" in name.attrib:
currency_names_plural.setdefault(code, {})[name.attrib["count"]] = text_type(name.text)
else:
currency_names[code] = text_type(name.text)
# TODO: support choice patterns for currency symbol selection
symbol = elem.find("symbol")
if symbol is not None and "draft" not in symbol.attrib and "choice" not in symbol.attrib:
currency_symbols[code] = text_type(symbol.text)
# <units>
unit_patterns = data.setdefault("unit_patterns", {})
for elem in tree.findall(".//units/unitLength"):
unit_length_type = elem.attrib["type"]
for unit in elem.findall("unit"):
unit_type = unit.attrib["type"]
for pattern in unit.findall("unitPattern"):
box = unit_type
box += ":" + unit_length_type
unit_patterns.setdefault(box, {})[pattern.attrib["count"]] = text_type(pattern.text)
date_fields = data.setdefault("date_fields", {})
for elem in tree.findall(".//dates/fields/field"):
field_type = elem.attrib["type"]
date_fields.setdefault(field_type, {})
for rel_time in elem.findall("relativeTime"):
rel_time_type = rel_time.attrib["type"]
for pattern in rel_time.findall("relativeTimePattern"):
date_fields[field_type].setdefault(rel_time_type, {})[pattern.attrib["count"]] = text_type(
pattern.text
)
outfile = open(data_filename, "wb")
try:
pickle.dump(data, outfile, 2)
finally:
outfile.close()
def test_parse_static_pattern():
assert numbers.parse_pattern('Kun') # in the So locale in CLDR 30
# <numbers>
number_symbols = data.setdefault('number_symbols', {})
for elem in tree.findall('.//numbers/symbols/*'):
if ('draft' in elem.attrib or 'alt' in elem.attrib):
continue
number_symbols[elem.tag] = unicode(elem.text)
decimal_formats = data.setdefault('decimal_formats', {})
for elem in tree.findall('.//decimalFormats/decimalFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in decimal_formats:
continue
pattern = unicode(elem.findtext('decimalFormat/pattern'))
decimal_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern)
scientific_formats = data.setdefault('scientific_formats', {})
for elem in tree.findall('.//scientificFormats/scientificFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in scientific_formats:
continue
pattern = unicode(elem.findtext('scientificFormat/pattern'))
scientific_formats[elem.attrib.get('type')] = numbers.parse_pattern(pattern)
currency_formats = data.setdefault('currency_formats', {})
for elem in tree.findall('.//currencyFormats/currencyFormatLength'):
if ('draft' in elem.attrib or 'alt' in elem.attrib) \
and elem.attrib.get('type') in currency_formats:
continue
pattern = unicode(elem.findtext('currencyFormat/pattern'))