Ejemplo n.º 1
0
def test_parse_pattern():
    assert dates.parse_pattern("MMMMd").format == u'%(MMMM)s%(d)s'
    assert (dates.parse_pattern("MMM d, yyyy").format ==
            u'%(MMM)s %(d)s, %(yyyy)s')
    assert (dates.parse_pattern("H:mm' Uhr 'z").format ==
            u'%(H)s:%(mm)s Uhr %(z)s')
    assert dates.parse_pattern("hh' o''clock'").format == u"%(hh)s o'clock"
Ejemplo n.º 2
0
    def processFormats(self, locale, formats):
        '''
        Process the formats to a complete list of formats that will be used by conversion.
        '''
        assert isinstance(formats, dict), 'Invalid formats %s' % formats
        assert isinstance(locale, Locale), 'Invalid locale %s' % locale

        for clsTyp, format in formats.items():
            # In here we just check that the format is valid.
            try:
                if clsTyp in (Number, Percentage): bn.parse_pattern(format)
                elif format not in self.formats[clsTyp]: bd.parse_pattern(format)
            except Exception as e:
                raise FormatError('invalid %s format \'%s\' because: %s' % (clsTyp.__name__, format, str(e)))

        if Number not in formats: formats[Number] = locale.decimal_formats.get(None).pattern
        if Percentage not in formats: formats[Percentage] = locale.percent_formats.get(None).pattern

        for clsTyp, default in self.defaults.items():
            if clsTyp not in formats: formats[clsTyp] = default

        return formats
Ejemplo n.º 3
0
def babel2datepicker(pattern):
  """
  Converts date format from babel
  (http://babel.pocoo.org/docs/dates/#date-fields)) to a format understood by
  bootstrap-datepicker.
  """
  if not isinstance(pattern, DateTimePattern):
    pattern = parse_pattern(pattern)

  map_fmt = {
    # days
    'd'    : 'dd',
    'dd'   : 'dd',
    'EEE'  : 'D',
    'EEEE' : 'DD',
    'EEEEE': 'D',  # narrow name => short name
    # months
    'M'   : 'mm',
    'MM'  : 'mm',
    'MMM' : 'M',
    'MMMM': 'MM',
    # years
    'y'   : 'yyyy',
    'yy'  : 'yyyy',
    'yyy' : 'yyyy',
    'yyyy': 'yyyy',

    # time picker format
    # hours
    'h' : '%I',
    'hh': '%I',
    'H' : '%H',
    'HH': '%H',
    # minutes,
    'm' : '%M',
    'mm': '%M',
    # seconds
    's' : '%S',
    'ss': '%S',
    # am/pm
    'a': '%p',

    }

  return pattern.format % map_fmt
Ejemplo n.º 4
0
def parse_calendar_time_formats(data, calendar):
    time_formats = data.setdefault('time_formats', {})
    for format in calendar.findall('timeFormats'):
        for elem in format.getiterator():
            if elem.tag == 'timeFormatLength':
                type = elem.attrib.get('type')
                if _should_skip_elem(elem, type, time_formats):
                    continue
                try:
                    time_formats[type] = dates.parse_pattern(
                        text_type(elem.findtext('timeFormat/pattern'))
                    )
                except ValueError as e:
                    error(e)
            elif elem.tag == 'alias':
                time_formats = Alias(_translate_alias(
                    ['time_formats'], elem.attrib['path'])
                )
Ejemplo n.º 5
0
def babel2datepicker(pattern):
    """Convert date format from babel (http://babel.pocoo.org/docs/dates/#date-
    fields)) to a format understood by bootstrap-datepicker."""
    if not isinstance(pattern, DateTimePattern):
        pattern = parse_pattern(pattern)

    map_fmt = {
        # days
        "d": "dd",
        "dd": "dd",
        "EEE": "D",
        "EEEE": "DD",
        "EEEEE": "D",  # narrow name => short name
        # months
        "M": "mm",
        "MM": "mm",
        "MMM": "M",
        "MMMM": "MM",
        # years
        "y": "yyyy",
        "yy": "yyyy",
        "yyy": "yyyy",
        "yyyy": "yyyy",
        # time picker format
        # hours
        "h": "%I",
        "hh": "%I",
        "H": "%H",
        "HH": "%H",
        # minutes,
        "m": "%M",
        "mm": "%M",
        # seconds
        "s": "%S",
        "ss": "%S",
        # am/pm
        "a": "%p",
    }

    return pattern.format % map_fmt
Ejemplo n.º 6
0
def parse_calendar_datetime_skeletons(data, calendar):
    datetime_formats = data.setdefault('datetime_formats', {})
    datetime_skeletons = data.setdefault('datetime_skeletons', {})
    for format in calendar.findall('dateTimeFormats'):
        for elem in format.getiterator():
            if elem.tag == 'dateTimeFormatLength':
                type = elem.attrib.get('type')
                if _should_skip_elem(elem, type, datetime_formats):
                    continue
                try:
                    datetime_formats[type] = text_type(elem.findtext('dateTimeFormat/pattern'))
                except ValueError as e:
                    error(e)
            elif elem.tag == 'alias':
                datetime_formats = Alias(_translate_alias(
                    ['datetime_formats'], elem.attrib['path'])
                )
            elif elem.tag == 'availableFormats':
                for datetime_skeleton in elem.findall('dateFormatItem'):
                    datetime_skeletons[datetime_skeleton.attrib['id']] = (
                        dates.parse_pattern(text_type(datetime_skeleton.text))
                    )
Ejemplo n.º 7
0
def main():
    parser = OptionParser(usage='%prog path/to/cldr')
    options, args = parser.parse_args()
    if len(args) != 1:
        parser.error('incorrect number of arguments')

    srcdir = args[0]
    destdir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),
                           '..', 'babel')

    sup_filename = os.path.join(srcdir, 'supplemental', 'supplementalData.xml')
    bcp47_timezone = parse(os.path.join(srcdir, 'bcp47', 'timezone.xml'))
    sup_windows_zones = parse(os.path.join(srcdir, 'supplemental',
                                           'windowsZones.xml'))
    sup_metadata = parse(os.path.join(srcdir, 'supplemental',
                                      'supplementalMetadata.xml'))
    sup_likely = parse(os.path.join(srcdir, 'supplemental',
                                    'likelySubtags.xml'))
    sup = parse(sup_filename)

    # Import global data from the supplemental files
    global_path = os.path.join(destdir, 'global.dat')
    global_data = {}
    if need_conversion(global_path, global_data, sup_filename):
        territory_zones = global_data.setdefault('territory_zones', {})
        zone_aliases = global_data.setdefault('zone_aliases', {})
        zone_territories = global_data.setdefault('zone_territories', {})
        win_mapping = global_data.setdefault('windows_zone_mapping', {})
        language_aliases = global_data.setdefault('language_aliases', {})
        territory_aliases = global_data.setdefault('territory_aliases', {})
        script_aliases = global_data.setdefault('script_aliases', {})
        variant_aliases = global_data.setdefault('variant_aliases', {})
        likely_subtags = global_data.setdefault('likely_subtags', {})
        territory_currencies = global_data.setdefault('territory_currencies', {})
        parent_exceptions = global_data.setdefault('parent_exceptions', {})

        # create auxiliary zone->territory map from the windows zones (we don't set
        # the 'zones_territories' map directly here, because there are some zones
        # aliases listed and we defer the decision of which ones to choose to the
        # 'bcp47' data
        _zone_territory_map = {}
        for map_zone in sup_windows_zones.findall(
                './/windowsZones/mapTimezones/mapZone'):
            if map_zone.attrib.get('territory') == '001':
                win_mapping[map_zone.attrib['other']] = \
                    map_zone.attrib['type'].split()[0]
            for tzid in text_type(map_zone.attrib['type']).split():
                _zone_territory_map[tzid] = \
                    text_type(map_zone.attrib['territory'])

        for key_elem in bcp47_timezone.findall('.//keyword/key'):
            if key_elem.attrib['name'] == 'tz':
                for elem in key_elem.findall('type'):
                    if 'deprecated' not in elem.attrib:
                        aliases = text_type(elem.attrib['alias']).split()
                        tzid = aliases.pop(0)
                        territory = _zone_territory_map.get(tzid, '001')
                        territory_zones.setdefault(territory, []).append(tzid)
                        zone_territories[tzid] = territory
                        for alias in aliases:
                            zone_aliases[alias] = tzid
                break

        # Import Metazone mapping
        meta_zones = global_data.setdefault('meta_zones', {})
        tzsup = parse(os.path.join(srcdir, 'supplemental', 'metaZones.xml'))
        for elem in tzsup.findall('.//timezone'):
            for child in elem.findall('usesMetazone'):
                if 'to' not in child.attrib: # FIXME: support old mappings
                    meta_zones[elem.attrib['type']] = child.attrib['mzone']

        # Language aliases
        for alias in sup_metadata.findall('.//alias/languageAlias'):
            # We don't have a use for those at the moment.  They don't
            # pass our parser anyways.
            if '_' in alias.attrib['type']:
                continue
            language_aliases[alias.attrib['type']] = alias.attrib['replacement']

        # Territory aliases
        for alias in sup_metadata.findall('.//alias/territoryAlias'):
            territory_aliases[alias.attrib['type']] = \
                alias.attrib['replacement'].split()

        # Script aliases
        for alias in sup_metadata.findall('.//alias/scriptAlias'):
            script_aliases[alias.attrib['type']] = alias.attrib['replacement']

        # Variant aliases
        for alias in sup_metadata.findall('.//alias/variantAlias'):
            repl = alias.attrib.get('replacement')
            if repl:
                variant_aliases[alias.attrib['type']] = repl

        # Likely subtags
        for likely_subtag in sup_likely.findall('.//likelySubtags/likelySubtag'):
            likely_subtags[likely_subtag.attrib['from']] = \
                likely_subtag.attrib['to']

        # Currencies in territories
        for region in sup.findall('.//currencyData/region'):
            region_code = region.attrib['iso3166']
            region_currencies = []
            for currency in region.findall('./currency'):
                cur_start = _parse_currency_date(currency.attrib.get('from'))
                cur_end = _parse_currency_date(currency.attrib.get('to'))
                region_currencies.append((currency.attrib['iso4217'],
                                          cur_start, cur_end,
                                          currency.attrib.get(
                                              'tender', 'true') == 'true'))
            region_currencies.sort(key=_currency_sort_key)
            territory_currencies[region_code] = region_currencies

        # Explicit parent locales
        for paternity in sup.findall('.//parentLocales/parentLocale'):
            parent = paternity.attrib['parent']
            for child in paternity.attrib['locales'].split():
                parent_exceptions[child] = parent

        outfile = open(global_path, 'wb')
        try:
            pickle.dump(global_data, outfile, 2)
        finally:
            outfile.close()

    # build a territory containment mapping for inheritance
    regions = {}
    for elem in sup.findall('.//territoryContainment/group'):
        regions[elem.attrib['type']] = elem.attrib['contains'].split()

    # Resolve territory containment
    territory_containment = {}
    region_items = sorted(regions.items())
    for group, territory_list in region_items:
        for territory in territory_list:
            containers = territory_containment.setdefault(territory, set([]))
            if group in territory_containment:
                containers |= territory_containment[group]
            containers.add(group)

    # prepare the per-locale plural rules definitions
    plural_rules = {}
    prsup = parse(os.path.join(srcdir, 'supplemental', 'plurals.xml'))
    for elem in prsup.findall('.//plurals/pluralRules'):
        rules = []
        for rule in elem.findall('pluralRule'):
            rules.append((rule.attrib['count'], text_type(rule.text)))
        pr = PluralRule(rules)
        for locale in elem.attrib['locales'].split():
            plural_rules[locale] = pr

    filenames = os.listdir(os.path.join(srcdir, 'main'))
    filenames.remove('root.xml')
    filenames.sort(key=len)
    filenames.insert(0, 'root.xml')

    for filename in filenames:
        stem, ext = os.path.splitext(filename)
        if ext != '.xml':
            continue

        full_filename = os.path.join(srcdir, 'main', filename)
        data_filename = os.path.join(destdir, 'localedata', stem + '.dat')

        data = {}
        if not need_conversion(data_filename, data, full_filename):
            continue

        tree = parse(full_filename)

        language = None
        elem = tree.find('.//identity/language')
        if elem is not None:
            language = elem.attrib['type']

        territory = None
        elem = tree.find('.//identity/territory')
        if elem is not None:
            territory = elem.attrib['type']
        else:
            territory = '001' # world
        regions = territory_containment.get(territory, [])

        log('Processing %s (Language = %s; Territory = %s)',
            filename, language, territory)

        # plural rules
        locale_id = '_'.join(filter(None, [
            language,
            territory != '001' and territory or None
        ]))
        if locale_id in plural_rules:
            data['plural_form'] = plural_rules[locale_id]

        # <localeDisplayNames>

        territories = data.setdefault('territories', {})
        for elem in tree.findall('.//territories/territory'):
            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                    and elem.attrib['type'] in territories:
                continue
            territories[elem.attrib['type']] = _text(elem)

        languages = data.setdefault('languages', {})
        for elem in tree.findall('.//languages/language'):
            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                    and elem.attrib['type'] in languages:
                continue
            languages[elem.attrib['type']] = _text(elem)

        variants = data.setdefault('variants', {})
        for elem in tree.findall('.//variants/variant'):
            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                    and elem.attrib['type'] in variants:
                continue
            variants[elem.attrib['type']] = _text(elem)

        scripts = data.setdefault('scripts', {})
        for elem in tree.findall('.//scripts/script'):
            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                    and elem.attrib['type'] in scripts:
                continue
            scripts[elem.attrib['type']] = _text(elem)

        # <dates>

        week_data = data.setdefault('week_data', {})
        supelem = sup.find('.//weekData')

        for elem in supelem.findall('minDays'):
            territories = elem.attrib['territories'].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data['min_days'] = int(elem.attrib['count'])

        for elem in supelem.findall('firstDay'):
            territories = elem.attrib['territories'].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data['first_day'] = weekdays[elem.attrib['day']]

        for elem in supelem.findall('weekendStart'):
            territories = elem.attrib['territories'].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data['weekend_start'] = weekdays[elem.attrib['day']]

        for elem in supelem.findall('weekendEnd'):
            territories = elem.attrib['territories'].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data['weekend_end'] = weekdays[elem.attrib['day']]

        zone_formats = data.setdefault('zone_formats', {})
        for elem in tree.findall('.//timeZoneNames/gmtFormat'):
            if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
                zone_formats['gmt'] = text_type(elem.text).replace('{0}', '%s')
                break
        for elem in tree.findall('.//timeZoneNames/regionFormat'):
            if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
                zone_formats['region'] = text_type(elem.text).replace('{0}', '%s')
                break
        for elem in tree.findall('.//timeZoneNames/fallbackFormat'):
            if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
                zone_formats['fallback'] = text_type(elem.text) \
                    .replace('{0}', '%(0)s').replace('{1}', '%(1)s')
                break
        for elem in tree.findall('.//timeZoneNames/fallbackRegionFormat'):
            if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
                zone_formats['fallback_region'] = text_type(elem.text) \
                    .replace('{0}', '%(0)s').replace('{1}', '%(1)s')
                break

        time_zones = data.setdefault('time_zones', {})
        for elem in tree.findall('.//timeZoneNames/zone'):
            info = {}
            city = elem.findtext('exemplarCity')
            if city:
                info['city'] = text_type(city)
            for child in elem.findall('long/*'):
                info.setdefault('long', {})[child.tag] = text_type(child.text)
            for child in elem.findall('short/*'):
                info.setdefault('short', {})[child.tag] = text_type(child.text)
            time_zones[elem.attrib['type']] = info

        meta_zones = data.setdefault('meta_zones', {})
        for elem in tree.findall('.//timeZoneNames/metazone'):
            info = {}
            city = elem.findtext('exemplarCity')
            if city:
                info['city'] = text_type(city)
            for child in elem.findall('long/*'):
                info.setdefault('long', {})[child.tag] = text_type(child.text)
            for child in elem.findall('short/*'):
                info.setdefault('short', {})[child.tag] = text_type(child.text)
            meta_zones[elem.attrib['type']] = info

        for calendar in tree.findall('.//calendars/calendar'):
            if calendar.attrib['type'] != 'gregorian':
                # TODO: support other calendar types
                continue

            months = data.setdefault('months', {})
            for ctxt in calendar.findall('months/monthContext'):
                ctxt_type = ctxt.attrib['type']
                ctxts = months.setdefault(ctxt_type, {})
                for width in ctxt.findall('monthWidth'):
                    width_type = width.attrib['type']
                    widths = ctxts.setdefault(width_type, {})
                    for elem in width.getiterator():
                        if elem.tag == 'month':
                            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                                    and int(elem.attrib['type']) in widths:
                                continue
                            widths[int(elem.attrib.get('type'))] = \
                                text_type(elem.text)
                        elif elem.tag == 'alias':
                            ctxts[width_type] = Alias(
                                _translate_alias(['months', ctxt_type, width_type],
                                                 elem.attrib['path'])
                            )

            days = data.setdefault('days', {})
            for ctxt in calendar.findall('days/dayContext'):
                ctxt_type = ctxt.attrib['type']
                ctxts = days.setdefault(ctxt_type, {})
                for width in ctxt.findall('dayWidth'):
                    width_type = width.attrib['type']
                    widths = ctxts.setdefault(width_type, {})
                    for elem in width.getiterator():
                        if elem.tag == 'day':
                            dtype = weekdays[elem.attrib['type']]
                            if ('draft' in elem.attrib or
                                'alt' not in elem.attrib) \
                                    and dtype in widths:
                                continue
                            widths[dtype] = text_type(elem.text)
                        elif elem.tag == 'alias':
                            ctxts[width_type] = Alias(
                                _translate_alias(['days', ctxt_type, width_type],
                                                 elem.attrib['path'])
                            )

            quarters = data.setdefault('quarters', {})
            for ctxt in calendar.findall('quarters/quarterContext'):
                ctxt_type = ctxt.attrib['type']
                ctxts = quarters.setdefault(ctxt.attrib['type'], {})
                for width in ctxt.findall('quarterWidth'):
                    width_type = width.attrib['type']
                    widths = ctxts.setdefault(width_type, {})
                    for elem in width.getiterator():
                        if elem.tag == 'quarter':
                            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                                    and int(elem.attrib['type']) in widths:
                                continue
                            widths[int(elem.attrib['type'])] = text_type(elem.text)
                        elif elem.tag == 'alias':
                            ctxts[width_type] = Alias(
                                _translate_alias(['quarters', ctxt_type,
                                                  width_type],
                                                 elem.attrib['path']))

            eras = data.setdefault('eras', {})
            for width in calendar.findall('eras/*'):
                width_type = NAME_MAP[width.tag]
                widths = eras.setdefault(width_type, {})
                for elem in width.getiterator():
                    if elem.tag == 'era':
                        if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                                and int(elem.attrib['type']) in widths:
                            continue
                        widths[int(elem.attrib.get('type'))] = text_type(elem.text)
                    elif elem.tag == 'alias':
                        eras[width_type] = Alias(
                            _translate_alias(['eras', width_type],
                                             elem.attrib['path'])
                        )

            # AM/PM
            periods = data.setdefault('periods', {})
            for day_period_width in calendar.findall(
                    'dayPeriods/dayPeriodContext/dayPeriodWidth'):
                if day_period_width.attrib['type'] == 'wide':
                    for day_period in day_period_width.findall('dayPeriod'):
                        if 'alt' not in day_period.attrib:
                            periods[day_period.attrib['type']] = text_type(
                                day_period.text)

            date_formats = data.setdefault('date_formats', {})
            for format in calendar.findall('dateFormats'):
                for elem in format.getiterator():
                    if elem.tag == 'dateFormatLength':
                        if 'draft' in elem.attrib and \
                                elem.attrib.get('type') in date_formats:
                            continue
                        try:
                            date_formats[elem.attrib.get('type')] = \
                                dates.parse_pattern(text_type(
                                    elem.findtext('dateFormat/pattern')))
                        except ValueError as e:
                            error(e)
                    elif elem.tag == 'alias':
                        date_formats = Alias(_translate_alias(
                            ['date_formats'], elem.attrib['path'])
                        )

            time_formats = data.setdefault('time_formats', {})
            for format in calendar.findall('timeFormats'):
                for elem in format.getiterator():
                    if elem.tag == 'timeFormatLength':
                        if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                                and elem.attrib.get('type') in time_formats:
                            continue
                        try:
                            time_formats[elem.attrib.get('type')] = \
                                dates.parse_pattern(text_type(
                                    elem.findtext('timeFormat/pattern')))
                        except ValueError as e:
                            error(e)
                    elif elem.tag == 'alias':
                        time_formats = Alias(_translate_alias(
                            ['time_formats'], elem.attrib['path'])
                        )

            datetime_formats = data.setdefault('datetime_formats', {})
            for format in calendar.findall('dateTimeFormats'):
                for elem in format.getiterator():
                    if elem.tag == 'dateTimeFormatLength':
                        if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                                and elem.attrib.get('type') in datetime_formats:
                            continue
                        try:
                            datetime_formats[elem.attrib.get('type')] = \
                                text_type(elem.findtext('dateTimeFormat/pattern'))
                        except ValueError as e:
                            error(e)
                    elif elem.tag == 'alias':
                        datetime_formats = Alias(_translate_alias(
                            ['datetime_formats'], elem.attrib['path'])
                        )

        # <numbers>

        number_symbols = data.setdefault('number_symbols', {})
        for elem in tree.findall('.//numbers/symbols/*'):
            if ('draft' in elem.attrib or 'alt' in elem.attrib):
                continue
            number_symbols[elem.tag] = text_type(elem.text)

        decimal_formats = data.setdefault('decimal_formats', {})
        for elem in tree.findall('.//decimalFormats/decimalFormatLength'):
            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                    and elem.attrib.get('type') in decimal_formats:
                continue
            if elem.findall('./alias'):
                # TODO map the alias to its target
                continue
            pattern = text_type(elem.findtext('./decimalFormat/pattern'))
            decimal_formats[elem.attrib.get('type')] = \
                numbers.parse_pattern(pattern)

        scientific_formats = data.setdefault('scientific_formats', {})
        for elem in tree.findall('.//scientificFormats/scientificFormatLength'):
            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                    and elem.attrib.get('type') in scientific_formats:
                continue
            pattern = text_type(elem.findtext('scientificFormat/pattern'))
            scientific_formats[elem.attrib.get('type')] = \
                numbers.parse_pattern(pattern)

        currency_formats = data.setdefault('currency_formats', {})
        for elem in tree.findall('.//currencyFormats/currencyFormatLength'):
            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                    and elem.attrib.get('type') in currency_formats:
                continue
            pattern = text_type(elem.findtext('currencyFormat/pattern'))
            currency_formats[elem.attrib.get('type')] = \
                numbers.parse_pattern(pattern)

        percent_formats = data.setdefault('percent_formats', {})
        for elem in tree.findall('.//percentFormats/percentFormatLength'):
            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                    and elem.attrib.get('type') in percent_formats:
                continue
            pattern = text_type(elem.findtext('percentFormat/pattern'))
            percent_formats[elem.attrib.get('type')] = \
                numbers.parse_pattern(pattern)

        currency_names = data.setdefault('currency_names', {})
        currency_names_plural = data.setdefault('currency_names_plural', {})
        currency_symbols = data.setdefault('currency_symbols', {})
        for elem in tree.findall('.//currencies/currency'):
            code = elem.attrib['type']
            for name in elem.findall('displayName'):
                if ('draft' in name.attrib) and code in currency_names:
                    continue
                if 'count' in name.attrib:
                    currency_names_plural.setdefault(code, {})[
                        name.attrib['count']] = text_type(name.text)
                else:
                    currency_names[code] = text_type(name.text)
            # TODO: support choice patterns for currency symbol selection
            symbol = elem.find('symbol')
            if symbol is not None and 'draft' not in symbol.attrib \
                    and 'choice' not in symbol.attrib:
                currency_symbols[code] = text_type(symbol.text)

        # <units>

        unit_patterns = data.setdefault('unit_patterns', {})
        for elem in tree.findall('.//units/unitLength'):
            unit_length_type = elem.attrib['type']
            for unit in elem.findall('unit'):
                unit_type = unit.attrib['type']
                for pattern in unit.findall('unitPattern'):
                    box = unit_type
                    box += ':' + unit_length_type
                    unit_patterns.setdefault(box, {})[pattern.attrib['count']] = \
                        text_type(pattern.text)

        date_fields = data.setdefault('date_fields', {})
        for elem in tree.findall('.//dates/fields/field'):
            field_type = elem.attrib['type']
            date_fields.setdefault(field_type, {})
            for rel_time in elem.findall('relativeTime'):
                rel_time_type = rel_time.attrib['type']
                for pattern in rel_time.findall('relativeTimePattern'):
                    date_fields[field_type].setdefault(rel_time_type, {})\
                        [pattern.attrib['count']] = text_type(pattern.text)

        outfile = open(data_filename, 'wb')
        try:
            pickle.dump(data, outfile, 2)
        finally:
            outfile.close()
Ejemplo n.º 8
0
def main():
    parser = OptionParser(usage='%prog path/to/cldr')
    options, args = parser.parse_args()
    if len(args) != 1:
        parser.error('incorrect number of arguments')

    srcdir = args[0]
    destdir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),
                           '..', 'babel')

    sup = parse(os.path.join(srcdir, 'supplemental', 'supplementalData.xml'))

    # Import global data from the supplemental files
    global_data = {}

    territory_zones = global_data.setdefault('territory_zones', {})
    zone_aliases = global_data.setdefault('zone_aliases', {})
    zone_territories = global_data.setdefault('zone_territories', {})
    for elem in sup.findall('.//timezoneData/zoneFormatting/zoneItem'):
        tzid = elem.attrib['type']
        territory_zones.setdefault(elem.attrib['territory'], []).append(tzid)
        zone_territories[tzid] = elem.attrib['territory']
        if 'aliases' in elem.attrib:
            for alias in elem.attrib['aliases'].split():
                zone_aliases[alias] = tzid

    # Import Metazone mapping
    meta_zones = global_data.setdefault('meta_zones', {})
    tzsup = parse(os.path.join(srcdir, 'supplemental', 'metazoneInfo.xml'))
    for elem in tzsup.findall('.//timezone'):
        for child in elem.findall('usesMetazone'):
            if 'to' not in child.attrib: # FIXME: support old mappings
                meta_zones[elem.attrib['type']] = child.attrib['mzone']

    outfile = open(os.path.join(destdir, 'global.dat'), 'wb')
    try:
        pickle.dump(global_data, outfile, 2)
    finally:
        outfile.close()

    # build a territory containment mapping for inheritance
    regions = {}
    for elem in sup.findall('.//territoryContainment/group'):
        regions[elem.attrib['type']] = elem.attrib['contains'].split()

    # Resolve territory containment
    territory_containment = {}
    region_items = regions.items()
    region_items.sort()
    for group, territory_list in region_items:
        for territory in territory_list:
            containers = territory_containment.setdefault(territory, set([]))
            if group in territory_containment:
                containers |= territory_containment[group]
            containers.add(group)

    # prepare the per-locale plural rules definitions
    plural_rules = {}
    prsup = parse(os.path.join(srcdir, 'supplemental', 'plurals.xml'))
    for elem in prsup.findall('.//plurals/pluralRules'):
        rules = []
        for rule in elem.findall('pluralRule'):
            rules.append((rule.attrib['count'], unicode(rule.text)))
        pr = PluralRule(rules)
        for locale in elem.attrib['locales'].split():
            plural_rules[locale] = pr

    filenames = os.listdir(os.path.join(srcdir, 'main'))
    filenames.remove('root.xml')
    filenames.sort(lambda a,b: len(a)-len(b))
    filenames.insert(0, 'root.xml')

    for filename in filenames:
        stem, ext = os.path.splitext(filename)
        if ext != '.xml':
            continue

        print>>sys.stderr, 'Processing input file %r' % filename
        tree = parse(os.path.join(srcdir, 'main', filename))
        data = {}

        language = None
        elem = tree.find('.//identity/language')
        if elem is not None:
            language = elem.attrib['type']
        print>>sys.stderr, '  Language:  %r' % language

        territory = None
        elem = tree.find('.//identity/territory')
        if elem is not None:
            territory = elem.attrib['type']
        else:
            territory = '001' # world
        print>>sys.stderr, '  Territory: %r' % territory
        regions = territory_containment.get(territory, [])
        print>>sys.stderr, '  Regions:    %r' % regions

        # plural rules
        locale_id = '_'.join(filter(None, [
            language,
            territory != '001' and territory or None
        ]))
        if locale_id in plural_rules:
            data['plural_form'] = plural_rules[locale_id]

        # <localeDisplayNames>

        territories = data.setdefault('territories', {})
        for elem in tree.findall('.//territories/territory'):
            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                    and elem.attrib['type'] in territories:
                continue
            territories[elem.attrib['type']] = _text(elem)

        languages = data.setdefault('languages', {})
        for elem in tree.findall('.//languages/language'):
            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                    and elem.attrib['type'] in languages:
                continue
            languages[elem.attrib['type']] = _text(elem)

        variants = data.setdefault('variants', {})
        for elem in tree.findall('.//variants/variant'):
            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                    and elem.attrib['type'] in variants:
                continue
            variants[elem.attrib['type']] = _text(elem)

        scripts = data.setdefault('scripts', {})
        for elem in tree.findall('.//scripts/script'):
            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                    and elem.attrib['type'] in scripts:
                continue
            scripts[elem.attrib['type']] = _text(elem)

        # <dates>

        week_data = data.setdefault('week_data', {})
        supelem = sup.find('.//weekData')

        for elem in supelem.findall('minDays'):
            territories = elem.attrib['territories'].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data['min_days'] = int(elem.attrib['count'])

        for elem in supelem.findall('firstDay'):
            territories = elem.attrib['territories'].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data['first_day'] = weekdays[elem.attrib['day']]

        for elem in supelem.findall('weekendStart'):
            territories = elem.attrib['territories'].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data['weekend_start'] = weekdays[elem.attrib['day']]

        for elem in supelem.findall('weekendEnd'):
            territories = elem.attrib['territories'].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data['weekend_end'] = weekdays[elem.attrib['day']]

        zone_formats = data.setdefault('zone_formats', {})
        for elem in tree.findall('.//timeZoneNames/gmtFormat'):
            if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
                zone_formats['gmt'] = unicode(elem.text).replace('{0}', '%s')
                break
        for elem in tree.findall('.//timeZoneNames/regionFormat'):
            if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
                zone_formats['region'] = unicode(elem.text).replace('{0}', '%s')
                break
        for elem in tree.findall('.//timeZoneNames/fallbackFormat'):
            if 'draft' not in elem.attrib and 'alt' not in elem.attrib:
                zone_formats['fallback'] = unicode(elem.text) \
                    .replace('{0}', '%(0)s').replace('{1}', '%(1)s')
                break

        time_zones = data.setdefault('time_zones', {})
        for elem in tree.findall('.//timeZoneNames/zone'):
            info = {}
            city = elem.findtext('exemplarCity')
            if city:
                info['city'] = unicode(city)
            for child in elem.findall('long/*'):
                info.setdefault('long', {})[child.tag] = unicode(child.text)
            for child in elem.findall('short/*'):
                info.setdefault('short', {})[child.tag] = unicode(child.text)
            time_zones[elem.attrib['type']] = info

        meta_zones = data.setdefault('meta_zones', {})
        for elem in tree.findall('.//timeZoneNames/metazone'):
            info = {}
            city = elem.findtext('exemplarCity')
            if city:
                info['city'] = unicode(city)
            for child in elem.findall('long/*'):
                info.setdefault('long', {})[child.tag] = unicode(child.text)
            for child in elem.findall('short/*'):
                info.setdefault('short', {})[child.tag] = unicode(child.text)
            info['common'] = elem.findtext('commonlyUsed') == 'true'
            meta_zones[elem.attrib['type']] = info

        for calendar in tree.findall('.//calendars/calendar'):
            if calendar.attrib['type'] != 'gregorian':
                # TODO: support other calendar types
                continue

            months = data.setdefault('months', {})
            for ctxt in calendar.findall('months/monthContext'):
                ctxt_type = ctxt.attrib['type']
                ctxts = months.setdefault(ctxt_type, {})
                for width in ctxt.findall('monthWidth'):
                    width_type = width.attrib['type']
                    widths = ctxts.setdefault(width_type, {})
                    for elem in width.getiterator():
                        if elem.tag == 'month':
                            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                                    and int(elem.attrib['type']) in widths:
                                continue
                            widths[int(elem.attrib.get('type'))] = unicode(elem.text)
                        elif elem.tag == 'alias':
                            ctxts[width_type] = Alias(
                                _translate_alias(['months', ctxt_type, width_type],
                                                 elem.attrib['path'])
                            )

            days = data.setdefault('days', {})
            for ctxt in calendar.findall('days/dayContext'):
                ctxt_type = ctxt.attrib['type']
                ctxts = days.setdefault(ctxt_type, {})
                for width in ctxt.findall('dayWidth'):
                    width_type = width.attrib['type']
                    widths = ctxts.setdefault(width_type, {})
                    for elem in width.getiterator():
                        if elem.tag == 'day':
                            dtype = weekdays[elem.attrib['type']]
                            if ('draft' in elem.attrib or 'alt' not in elem.attrib) \
                                    and dtype in widths:
                                continue
                            widths[dtype] = unicode(elem.text)
                        elif elem.tag == 'alias':
                            ctxts[width_type] = Alias(
                                _translate_alias(['days', ctxt_type, width_type],
                                                 elem.attrib['path'])
                            )

            quarters = data.setdefault('quarters', {})
            for ctxt in calendar.findall('quarters/quarterContext'):
                ctxt_type = ctxt.attrib['type']
                ctxts = quarters.setdefault(ctxt.attrib['type'], {})
                for width in ctxt.findall('quarterWidth'):
                    width_type = width.attrib['type']
                    widths = ctxts.setdefault(width_type, {})
                    for elem in width.getiterator():
                        if elem.tag == 'quarter':
                            if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                                    and int(elem.attrib['type']) in widths:
                                continue
                            widths[int(elem.attrib['type'])] = unicode(elem.text)
                        elif elem.tag == 'alias':
                            ctxts[width_type] = Alias(
                                _translate_alias(['quarters', ctxt_type, width_type],
                                                 elem.attrib['path'])
                            )

            eras = data.setdefault('eras', {})
            for width in calendar.findall('eras/*'):
                width_type = NAME_MAP[width.tag]
                widths = eras.setdefault(width_type, {})
                for elem in width.getiterator():
                    if elem.tag == 'era':
                        if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                                and int(elem.attrib['type']) in widths:
                            continue
                        widths[int(elem.attrib.get('type'))] = unicode(elem.text)
                    elif elem.tag == 'alias':
                        eras[width_type] = Alias(
                            _translate_alias(['eras', width_type],
                                             elem.attrib['path'])
                        )

            # AM/PM
            periods = data.setdefault('periods', {})
            for elem in calendar.findall('am'):
                if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                        and elem.tag in periods:
                    continue
                periods[elem.tag] = unicode(elem.text)
            for elem in calendar.findall('pm'):
                if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                        and elem.tag in periods:
                    continue
                periods[elem.tag] = unicode(elem.text)

            date_formats = data.setdefault('date_formats', {})
            for format in calendar.findall('dateFormats'):
                for elem in format.getiterator():
                    if elem.tag == 'dateFormatLength':
                        if 'draft' in elem.attrib and \
                                elem.attrib.get('type') in date_formats:
                            continue
                        try:
                            date_formats[elem.attrib.get('type')] = \
                                dates.parse_pattern(unicode(elem.findtext('dateFormat/pattern')))
                        except ValueError, e:
                            print>>sys.stderr, 'ERROR: %s' % e
                    elif elem.tag == 'alias':
                        date_formats = Alias(_translate_alias(
                            ['date_formats'], elem.attrib['path'])
                        )

            time_formats = data.setdefault('time_formats', {})
            for format in calendar.findall('timeFormats'):
                for elem in format.getiterator():
                    if elem.tag == 'timeFormatLength':
                        if ('draft' in elem.attrib or 'alt' in elem.attrib) \
                                and elem.attrib.get('type') in time_formats:
                            continue
                        try:
                            time_formats[elem.attrib.get('type')] = \
                                dates.parse_pattern(unicode(elem.findtext('timeFormat/pattern')))
                        except ValueError, e:
                            print>>sys.stderr, 'ERROR: %s' % e
                    elif elem.tag == 'alias':
                        time_formats = Alias(_translate_alias(
                            ['time_formats'], elem.attrib['path'])
                        )
Ejemplo n.º 9
0
def main():
    parser = OptionParser(usage="%prog path/to/cldr")
    options, args = parser.parse_args()
    if len(args) != 1:
        parser.error("incorrect number of arguments")

    srcdir = args[0]
    destdir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "..", "babel")

    sup = parse(os.path.join(srcdir, "supplemental", "supplementalData.xml"))

    # Import global data from the supplemental files
    global_data = {}

    territory_zones = global_data.setdefault("territory_zones", {})
    zone_aliases = global_data.setdefault("zone_aliases", {})
    zone_territories = global_data.setdefault("zone_territories", {})
    for elem in sup.findall(".//timezoneData/zoneFormatting/zoneItem"):
        tzid = elem.attrib["type"]
        territory_zones.setdefault(elem.attrib["territory"], []).append(tzid)
        zone_territories[tzid] = elem.attrib["territory"]
        if "aliases" in elem.attrib:
            for alias in elem.attrib["aliases"].split():
                zone_aliases[alias] = tzid

    # Import Metazone mapping
    meta_zones = global_data.setdefault("meta_zones", {})
    tzsup = parse(os.path.join(srcdir, "supplemental", "metazoneInfo.xml"))
    for elem in tzsup.findall(".//timezone"):
        for child in elem.findall("usesMetazone"):
            if "to" not in child.attrib:  # FIXME: support old mappings
                meta_zones[elem.attrib["type"]] = child.attrib["mzone"]

    outfile = open(os.path.join(destdir, "global.dat"), "wb")
    try:
        pickle.dump(global_data, outfile, 2)
    finally:
        outfile.close()

    # build a territory containment mapping for inheritance
    regions = {}
    for elem in sup.findall(".//territoryContainment/group"):
        regions[elem.attrib["type"]] = elem.attrib["contains"].split()

    # Resolve territory containment
    territory_containment = {}
    region_items = regions.items()
    region_items.sort()
    for group, territory_list in region_items:
        for territory in territory_list:
            containers = territory_containment.setdefault(territory, set([]))
            if group in territory_containment:
                containers |= territory_containment[group]
            containers.add(group)

    # prepare the per-locale plural rules definitions
    plural_rules = {}
    prsup = parse(os.path.join(srcdir, "supplemental", "plurals.xml"))
    for elem in prsup.findall(".//plurals/pluralRules"):
        rules = []
        for rule in elem.findall("pluralRule"):
            rules.append((rule.attrib["count"], unicode(rule.text)))
        pr = PluralRule(rules)
        for locale in elem.attrib["locales"].split():
            plural_rules[locale] = pr

    filenames = os.listdir(os.path.join(srcdir, "main"))
    filenames.remove("root.xml")
    filenames.sort(lambda a, b: len(a) - len(b))
    filenames.insert(0, "root.xml")

    for filename in filenames:
        stem, ext = os.path.splitext(filename)
        if ext != ".xml":
            continue

        print >>sys.stderr, "Processing input file %r" % filename
        tree = parse(os.path.join(srcdir, "main", filename))
        data = {}

        language = None
        elem = tree.find(".//identity/language")
        if elem is not None:
            language = elem.attrib["type"]
        print >>sys.stderr, "  Language:  %r" % language

        territory = None
        elem = tree.find(".//identity/territory")
        if elem is not None:
            territory = elem.attrib["type"]
        else:
            territory = "001"  # world
        print >>sys.stderr, "  Territory: %r" % territory
        regions = territory_containment.get(territory, [])
        print >>sys.stderr, "  Regions:    %r" % regions

        # plural rules
        locale_id = "_".join(filter(None, [language, territory != "001" and territory or None]))
        if locale_id in plural_rules:
            data["plural_form"] = plural_rules[locale_id]

        # <localeDisplayNames>

        territories = data.setdefault("territories", {})
        for elem in tree.findall(".//territories/territory"):
            if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib["type"] in territories:
                continue
            territories[elem.attrib["type"]] = _text(elem)

        languages = data.setdefault("languages", {})
        for elem in tree.findall(".//languages/language"):
            if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib["type"] in languages:
                continue
            languages[elem.attrib["type"]] = _text(elem)

        variants = data.setdefault("variants", {})
        for elem in tree.findall(".//variants/variant"):
            if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib["type"] in variants:
                continue
            variants[elem.attrib["type"]] = _text(elem)

        scripts = data.setdefault("scripts", {})
        for elem in tree.findall(".//scripts/script"):
            if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib["type"] in scripts:
                continue
            scripts[elem.attrib["type"]] = _text(elem)

        # <dates>

        week_data = data.setdefault("week_data", {})
        supelem = sup.find(".//weekData")

        for elem in supelem.findall("minDays"):
            territories = elem.attrib["territories"].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data["min_days"] = int(elem.attrib["count"])

        for elem in supelem.findall("firstDay"):
            territories = elem.attrib["territories"].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data["first_day"] = weekdays[elem.attrib["day"]]

        for elem in supelem.findall("weekendStart"):
            territories = elem.attrib["territories"].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data["weekend_start"] = weekdays[elem.attrib["day"]]

        for elem in supelem.findall("weekendEnd"):
            territories = elem.attrib["territories"].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data["weekend_end"] = weekdays[elem.attrib["day"]]

        zone_formats = data.setdefault("zone_formats", {})
        for elem in tree.findall(".//timeZoneNames/gmtFormat"):
            if "draft" not in elem.attrib and "alt" not in elem.attrib:
                zone_formats["gmt"] = unicode(elem.text).replace("{0}", "%s")
                break
        for elem in tree.findall(".//timeZoneNames/regionFormat"):
            if "draft" not in elem.attrib and "alt" not in elem.attrib:
                zone_formats["region"] = unicode(elem.text).replace("{0}", "%s")
                break
        for elem in tree.findall(".//timeZoneNames/fallbackFormat"):
            if "draft" not in elem.attrib and "alt" not in elem.attrib:
                zone_formats["fallback"] = unicode(elem.text).replace("{0}", "%(0)s").replace("{1}", "%(1)s")
                break

        time_zones = data.setdefault("time_zones", {})
        for elem in tree.findall(".//timeZoneNames/zone"):
            info = {}
            city = elem.findtext("exemplarCity")
            if city:
                info["city"] = unicode(city)
            for child in elem.findall("long/*"):
                info.setdefault("long", {})[child.tag] = unicode(child.text)
            for child in elem.findall("short/*"):
                info.setdefault("short", {})[child.tag] = unicode(child.text)
            time_zones[elem.attrib["type"]] = info

        meta_zones = data.setdefault("meta_zones", {})
        for elem in tree.findall(".//timeZoneNames/metazone"):
            info = {}
            city = elem.findtext("exemplarCity")
            if city:
                info["city"] = unicode(city)
            for child in elem.findall("long/*"):
                info.setdefault("long", {})[child.tag] = unicode(child.text)
            for child in elem.findall("short/*"):
                info.setdefault("short", {})[child.tag] = unicode(child.text)
            info["common"] = elem.findtext("commonlyUsed") == "true"
            meta_zones[elem.attrib["type"]] = info

        for calendar in tree.findall(".//calendars/calendar"):
            if calendar.attrib["type"] != "gregorian":
                # TODO: support other calendar types
                continue

            months = data.setdefault("months", {})
            for ctxt in calendar.findall("months/monthContext"):
                ctxt_type = ctxt.attrib["type"]
                ctxts = months.setdefault(ctxt_type, {})
                for width in ctxt.findall("monthWidth"):
                    width_type = width.attrib["type"]
                    widths = ctxts.setdefault(width_type, {})
                    for elem in width.getiterator():
                        if elem.tag == "month":
                            if ("draft" in elem.attrib or "alt" in elem.attrib) and int(elem.attrib["type"]) in widths:
                                continue
                            widths[int(elem.attrib.get("type"))] = unicode(elem.text)
                        elif elem.tag == "alias":
                            ctxts[width_type] = Alias(
                                _translate_alias(["months", ctxt_type, width_type], elem.attrib["path"])
                            )

            days = data.setdefault("days", {})
            for ctxt in calendar.findall("days/dayContext"):
                ctxt_type = ctxt.attrib["type"]
                ctxts = days.setdefault(ctxt_type, {})
                for width in ctxt.findall("dayWidth"):
                    width_type = width.attrib["type"]
                    widths = ctxts.setdefault(width_type, {})
                    for elem in width.getiterator():
                        if elem.tag == "day":
                            dtype = weekdays[elem.attrib["type"]]
                            if ("draft" in elem.attrib or "alt" not in elem.attrib) and dtype in widths:
                                continue
                            widths[dtype] = unicode(elem.text)
                        elif elem.tag == "alias":
                            ctxts[width_type] = Alias(
                                _translate_alias(["days", ctxt_type, width_type], elem.attrib["path"])
                            )

            quarters = data.setdefault("quarters", {})
            for ctxt in calendar.findall("quarters/quarterContext"):
                ctxt_type = ctxt.attrib["type"]
                ctxts = quarters.setdefault(ctxt.attrib["type"], {})
                for width in ctxt.findall("quarterWidth"):
                    width_type = width.attrib["type"]
                    widths = ctxts.setdefault(width_type, {})
                    for elem in width.getiterator():
                        if elem.tag == "quarter":
                            if ("draft" in elem.attrib or "alt" in elem.attrib) and int(elem.attrib["type"]) in widths:
                                continue
                            widths[int(elem.attrib["type"])] = unicode(elem.text)
                        elif elem.tag == "alias":
                            ctxts[width_type] = Alias(
                                _translate_alias(["quarters", ctxt_type, width_type], elem.attrib["path"])
                            )

            eras = data.setdefault("eras", {})
            for width in calendar.findall("eras/*"):
                width_type = NAME_MAP[width.tag]
                widths = eras.setdefault(width_type, {})
                for elem in width.getiterator():
                    if elem.tag == "era":
                        if ("draft" in elem.attrib or "alt" in elem.attrib) and int(elem.attrib["type"]) in widths:
                            continue
                        widths[int(elem.attrib.get("type"))] = unicode(elem.text)
                    elif elem.tag == "alias":
                        eras[width_type] = Alias(_translate_alias(["eras", width_type], elem.attrib["path"]))

            # AM/PM
            periods = data.setdefault("periods", {})
            for elem in calendar.findall("am"):
                if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.tag in periods:
                    continue
                periods[elem.tag] = unicode(elem.text)
            for elem in calendar.findall("pm"):
                if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.tag in periods:
                    continue
                periods[elem.tag] = unicode(elem.text)

            date_formats = data.setdefault("date_formats", {})
            for format in calendar.findall("dateFormats"):
                for elem in format.getiterator():
                    if elem.tag == "dateFormatLength":
                        if "draft" in elem.attrib and elem.attrib.get("type") in date_formats:
                            continue
                        try:
                            date_formats[elem.attrib.get("type")] = dates.parse_pattern(
                                unicode(elem.findtext("dateFormat/pattern"))
                            )
                        except ValueError, e:
                            print >>sys.stderr, "ERROR: %s" % e
                    elif elem.tag == "alias":
                        date_formats = Alias(_translate_alias(["date_formats"], elem.attrib["path"]))

            time_formats = data.setdefault("time_formats", {})
            for format in calendar.findall("timeFormats"):
                for elem in format.getiterator():
                    if elem.tag == "timeFormatLength":
                        if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib.get("type") in time_formats:
                            continue
                        try:
                            time_formats[elem.attrib.get("type")] = dates.parse_pattern(
                                unicode(elem.findtext("timeFormat/pattern"))
                            )
                        except ValueError, e:
                            print >>sys.stderr, "ERROR: %s" % e
                    elif elem.tag == "alias":
                        time_formats = Alias(_translate_alias(["time_formats"], elem.attrib["path"]))
Ejemplo n.º 10
0
def main():
    parser = OptionParser(usage="%prog path/to/cldr")
    options, args = parser.parse_args()
    if len(args) != 1:
        parser.error("incorrect number of arguments")

    srcdir = args[0]
    destdir = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "..", "babel")

    sup_filename = os.path.join(srcdir, "supplemental", "supplementalData.xml")
    bcp47_timezone = parse(os.path.join(srcdir, "bcp47", "timezone.xml"))
    sup_windows_zones = parse(os.path.join(srcdir, "supplemental", "windowsZones.xml"))
    sup_metadata = parse(os.path.join(srcdir, "supplemental", "supplementalMetadata.xml"))
    sup_likely = parse(os.path.join(srcdir, "supplemental", "likelySubtags.xml"))
    sup = parse(sup_filename)

    # Import global data from the supplemental files
    global_path = os.path.join(destdir, "global.dat")
    global_data = {}
    if need_conversion(global_path, global_data, sup_filename):
        territory_zones = global_data.setdefault("territory_zones", {})
        zone_aliases = global_data.setdefault("zone_aliases", {})
        zone_territories = global_data.setdefault("zone_territories", {})
        win_mapping = global_data.setdefault("windows_zone_mapping", {})
        language_aliases = global_data.setdefault("language_aliases", {})
        territory_aliases = global_data.setdefault("territory_aliases", {})
        script_aliases = global_data.setdefault("script_aliases", {})
        variant_aliases = global_data.setdefault("variant_aliases", {})
        likely_subtags = global_data.setdefault("likely_subtags", {})
        territory_currencies = global_data.setdefault("territory_currencies", {})
        parent_exceptions = global_data.setdefault("parent_exceptions", {})
        currency_fractions = global_data.setdefault("currency_fractions", {})

        # create auxiliary zone->territory map from the windows zones (we don't set
        # the 'zones_territories' map directly here, because there are some zones
        # aliases listed and we defer the decision of which ones to choose to the
        # 'bcp47' data
        _zone_territory_map = {}
        for map_zone in sup_windows_zones.findall(".//windowsZones/mapTimezones/mapZone"):
            if map_zone.attrib.get("territory") == "001":
                win_mapping[map_zone.attrib["other"]] = map_zone.attrib["type"].split()[0]
            for tzid in text_type(map_zone.attrib["type"]).split():
                _zone_territory_map[tzid] = text_type(map_zone.attrib["territory"])

        for key_elem in bcp47_timezone.findall(".//keyword/key"):
            if key_elem.attrib["name"] == "tz":
                for elem in key_elem.findall("type"):
                    if "deprecated" not in elem.attrib:
                        aliases = text_type(elem.attrib["alias"]).split()
                        tzid = aliases.pop(0)
                        territory = _zone_territory_map.get(tzid, "001")
                        territory_zones.setdefault(territory, []).append(tzid)
                        zone_territories[tzid] = territory
                        for alias in aliases:
                            zone_aliases[alias] = tzid
                break

        # Import Metazone mapping
        meta_zones = global_data.setdefault("meta_zones", {})
        tzsup = parse(os.path.join(srcdir, "supplemental", "metaZones.xml"))
        for elem in tzsup.findall(".//timezone"):
            for child in elem.findall("usesMetazone"):
                if "to" not in child.attrib:  # FIXME: support old mappings
                    meta_zones[elem.attrib["type"]] = child.attrib["mzone"]

        # Language aliases
        for alias in sup_metadata.findall(".//alias/languageAlias"):
            # We don't have a use for those at the moment.  They don't
            # pass our parser anyways.
            if "_" in alias.attrib["type"]:
                continue
            language_aliases[alias.attrib["type"]] = alias.attrib["replacement"]

        # Territory aliases
        for alias in sup_metadata.findall(".//alias/territoryAlias"):
            territory_aliases[alias.attrib["type"]] = alias.attrib["replacement"].split()

        # Script aliases
        for alias in sup_metadata.findall(".//alias/scriptAlias"):
            script_aliases[alias.attrib["type"]] = alias.attrib["replacement"]

        # Variant aliases
        for alias in sup_metadata.findall(".//alias/variantAlias"):
            repl = alias.attrib.get("replacement")
            if repl:
                variant_aliases[alias.attrib["type"]] = repl

        # Likely subtags
        for likely_subtag in sup_likely.findall(".//likelySubtags/likelySubtag"):
            likely_subtags[likely_subtag.attrib["from"]] = likely_subtag.attrib["to"]

        # Currencies in territories
        for region in sup.findall(".//currencyData/region"):
            region_code = region.attrib["iso3166"]
            region_currencies = []
            for currency in region.findall("./currency"):
                cur_start = _parse_currency_date(currency.attrib.get("from"))
                cur_end = _parse_currency_date(currency.attrib.get("to"))
                region_currencies.append(
                    (currency.attrib["iso4217"], cur_start, cur_end, currency.attrib.get("tender", "true") == "true")
                )
            region_currencies.sort(key=_currency_sort_key)
            territory_currencies[region_code] = region_currencies

        # Explicit parent locales
        for paternity in sup.findall(".//parentLocales/parentLocale"):
            parent = paternity.attrib["parent"]
            for child in paternity.attrib["locales"].split():
                parent_exceptions[child] = parent

        # Currency decimal and rounding digits
        for fraction in sup.findall(".//currencyData/fractions/info"):
            cur_code = fraction.attrib["iso4217"]
            cur_digits = int(fraction.attrib["digits"])
            cur_rounding = int(fraction.attrib["rounding"])
            cur_cdigits = int(fraction.attrib.get("cashDigits", cur_digits))
            cur_crounding = int(fraction.attrib.get("cashRounding", cur_rounding))
            currency_fractions[cur_code] = (cur_digits, cur_rounding, cur_cdigits, cur_crounding)

        outfile = open(global_path, "wb")
        try:
            pickle.dump(global_data, outfile, 2)
        finally:
            outfile.close()

    # build a territory containment mapping for inheritance
    regions = {}
    for elem in sup.findall(".//territoryContainment/group"):
        regions[elem.attrib["type"]] = elem.attrib["contains"].split()

    # Resolve territory containment
    territory_containment = {}
    region_items = sorted(regions.items())
    for group, territory_list in region_items:
        for territory in territory_list:
            containers = territory_containment.setdefault(territory, set([]))
            if group in territory_containment:
                containers |= territory_containment[group]
            containers.add(group)

    # prepare the per-locale plural rules definitions
    plural_rules = {}
    prsup = parse(os.path.join(srcdir, "supplemental", "plurals.xml"))
    for elem in prsup.findall(".//plurals/pluralRules"):
        rules = []
        for rule in elem.findall("pluralRule"):
            rules.append((rule.attrib["count"], text_type(rule.text)))
        pr = PluralRule(rules)
        for locale in elem.attrib["locales"].split():
            plural_rules[locale] = pr

    filenames = os.listdir(os.path.join(srcdir, "main"))
    filenames.remove("root.xml")
    filenames.sort(key=len)
    filenames.insert(0, "root.xml")

    for filename in filenames:
        stem, ext = os.path.splitext(filename)
        if ext != ".xml":
            continue

        full_filename = os.path.join(srcdir, "main", filename)
        data_filename = os.path.join(destdir, "locale-data", stem + ".dat")

        data = {}
        if not need_conversion(data_filename, data, full_filename):
            continue

        tree = parse(full_filename)

        language = None
        elem = tree.find(".//identity/language")
        if elem is not None:
            language = elem.attrib["type"]

        territory = None
        elem = tree.find(".//identity/territory")
        if elem is not None:
            territory = elem.attrib["type"]
        else:
            territory = "001"  # world
        regions = territory_containment.get(territory, [])

        log("Processing %s (Language = %s; Territory = %s)", filename, language, territory)

        # plural rules
        locale_id = "_".join(filter(None, [language, territory != "001" and territory or None]))
        if locale_id in plural_rules:
            data["plural_form"] = plural_rules[locale_id]

        # <localeDisplayNames>

        territories = data.setdefault("territories", {})
        for elem in tree.findall(".//territories/territory"):
            if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib["type"] in territories:
                continue
            territories[elem.attrib["type"]] = _text(elem)

        languages = data.setdefault("languages", {})
        for elem in tree.findall(".//languages/language"):
            if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib["type"] in languages:
                continue
            languages[elem.attrib["type"]] = _text(elem)

        variants = data.setdefault("variants", {})
        for elem in tree.findall(".//variants/variant"):
            if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib["type"] in variants:
                continue
            variants[elem.attrib["type"]] = _text(elem)

        scripts = data.setdefault("scripts", {})
        for elem in tree.findall(".//scripts/script"):
            if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib["type"] in scripts:
                continue
            scripts[elem.attrib["type"]] = _text(elem)

        # <dates>

        week_data = data.setdefault("week_data", {})
        supelem = sup.find(".//weekData")

        for elem in supelem.findall("minDays"):
            territories = elem.attrib["territories"].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data["min_days"] = int(elem.attrib["count"])

        for elem in supelem.findall("firstDay"):
            territories = elem.attrib["territories"].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data["first_day"] = weekdays[elem.attrib["day"]]

        for elem in supelem.findall("weekendStart"):
            territories = elem.attrib["territories"].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data["weekend_start"] = weekdays[elem.attrib["day"]]

        for elem in supelem.findall("weekendEnd"):
            territories = elem.attrib["territories"].split()
            if territory in territories or any([r in territories for r in regions]):
                week_data["weekend_end"] = weekdays[elem.attrib["day"]]

        zone_formats = data.setdefault("zone_formats", {})
        for elem in tree.findall(".//timeZoneNames/gmtFormat"):
            if "draft" not in elem.attrib and "alt" not in elem.attrib:
                zone_formats["gmt"] = text_type(elem.text).replace("{0}", "%s")
                break
        for elem in tree.findall(".//timeZoneNames/regionFormat"):
            if "draft" not in elem.attrib and "alt" not in elem.attrib:
                zone_formats["region"] = text_type(elem.text).replace("{0}", "%s")
                break
        for elem in tree.findall(".//timeZoneNames/fallbackFormat"):
            if "draft" not in elem.attrib and "alt" not in elem.attrib:
                zone_formats["fallback"] = text_type(elem.text).replace("{0}", "%(0)s").replace("{1}", "%(1)s")
                break
        for elem in tree.findall(".//timeZoneNames/fallbackRegionFormat"):
            if "draft" not in elem.attrib and "alt" not in elem.attrib:
                zone_formats["fallback_region"] = text_type(elem.text).replace("{0}", "%(0)s").replace("{1}", "%(1)s")
                break

        time_zones = data.setdefault("time_zones", {})
        for elem in tree.findall(".//timeZoneNames/zone"):
            info = {}
            city = elem.findtext("exemplarCity")
            if city:
                info["city"] = text_type(city)
            for child in elem.findall("long/*"):
                info.setdefault("long", {})[child.tag] = text_type(child.text)
            for child in elem.findall("short/*"):
                info.setdefault("short", {})[child.tag] = text_type(child.text)
            time_zones[elem.attrib["type"]] = info

        meta_zones = data.setdefault("meta_zones", {})
        for elem in tree.findall(".//timeZoneNames/metazone"):
            info = {}
            city = elem.findtext("exemplarCity")
            if city:
                info["city"] = text_type(city)
            for child in elem.findall("long/*"):
                info.setdefault("long", {})[child.tag] = text_type(child.text)
            for child in elem.findall("short/*"):
                info.setdefault("short", {})[child.tag] = text_type(child.text)
            meta_zones[elem.attrib["type"]] = info

        for calendar in tree.findall(".//calendars/calendar"):
            if calendar.attrib["type"] != "gregorian":
                # TODO: support other calendar types
                continue

            months = data.setdefault("months", {})
            for ctxt in calendar.findall("months/monthContext"):
                ctxt_type = ctxt.attrib["type"]
                ctxts = months.setdefault(ctxt_type, {})
                for width in ctxt.findall("monthWidth"):
                    width_type = width.attrib["type"]
                    widths = ctxts.setdefault(width_type, {})
                    for elem in width.getiterator():
                        if elem.tag == "month":
                            if ("draft" in elem.attrib or "alt" in elem.attrib) and int(elem.attrib["type"]) in widths:
                                continue
                            widths[int(elem.attrib.get("type"))] = text_type(elem.text)
                        elif elem.tag == "alias":
                            ctxts[width_type] = Alias(
                                _translate_alias(["months", ctxt_type, width_type], elem.attrib["path"])
                            )

            days = data.setdefault("days", {})
            for ctxt in calendar.findall("days/dayContext"):
                ctxt_type = ctxt.attrib["type"]
                ctxts = days.setdefault(ctxt_type, {})
                for width in ctxt.findall("dayWidth"):
                    width_type = width.attrib["type"]
                    widths = ctxts.setdefault(width_type, {})
                    for elem in width.getiterator():
                        if elem.tag == "day":
                            dtype = weekdays[elem.attrib["type"]]
                            if ("draft" in elem.attrib or "alt" not in elem.attrib) and dtype in widths:
                                continue
                            widths[dtype] = text_type(elem.text)
                        elif elem.tag == "alias":
                            ctxts[width_type] = Alias(
                                _translate_alias(["days", ctxt_type, width_type], elem.attrib["path"])
                            )

            quarters = data.setdefault("quarters", {})
            for ctxt in calendar.findall("quarters/quarterContext"):
                ctxt_type = ctxt.attrib["type"]
                ctxts = quarters.setdefault(ctxt.attrib["type"], {})
                for width in ctxt.findall("quarterWidth"):
                    width_type = width.attrib["type"]
                    widths = ctxts.setdefault(width_type, {})
                    for elem in width.getiterator():
                        if elem.tag == "quarter":
                            if ("draft" in elem.attrib or "alt" in elem.attrib) and int(elem.attrib["type"]) in widths:
                                continue
                            widths[int(elem.attrib["type"])] = text_type(elem.text)
                        elif elem.tag == "alias":
                            ctxts[width_type] = Alias(
                                _translate_alias(["quarters", ctxt_type, width_type], elem.attrib["path"])
                            )

            eras = data.setdefault("eras", {})
            for width in calendar.findall("eras/*"):
                width_type = NAME_MAP[width.tag]
                widths = eras.setdefault(width_type, {})
                for elem in width.getiterator():
                    if elem.tag == "era":
                        if ("draft" in elem.attrib or "alt" in elem.attrib) and int(elem.attrib["type"]) in widths:
                            continue
                        widths[int(elem.attrib.get("type"))] = text_type(elem.text)
                    elif elem.tag == "alias":
                        eras[width_type] = Alias(_translate_alias(["eras", width_type], elem.attrib["path"]))

            # AM/PM
            periods = data.setdefault("periods", {})
            for day_period_width in calendar.findall("dayPeriods/dayPeriodContext/dayPeriodWidth"):
                if day_period_width.attrib["type"] == "wide":
                    for day_period in day_period_width.findall("dayPeriod"):
                        if "alt" not in day_period.attrib:
                            periods[day_period.attrib["type"]] = text_type(day_period.text)

            date_formats = data.setdefault("date_formats", {})
            for format in calendar.findall("dateFormats"):
                for elem in format.getiterator():
                    if elem.tag == "dateFormatLength":
                        if "draft" in elem.attrib and elem.attrib.get("type") in date_formats:
                            continue
                        try:
                            date_formats[elem.attrib.get("type")] = dates.parse_pattern(
                                text_type(elem.findtext("dateFormat/pattern"))
                            )
                        except ValueError as e:
                            error(e)
                    elif elem.tag == "alias":
                        date_formats = Alias(_translate_alias(["date_formats"], elem.attrib["path"]))

            time_formats = data.setdefault("time_formats", {})
            for format in calendar.findall("timeFormats"):
                for elem in format.getiterator():
                    if elem.tag == "timeFormatLength":
                        if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib.get("type") in time_formats:
                            continue
                        try:
                            time_formats[elem.attrib.get("type")] = dates.parse_pattern(
                                text_type(elem.findtext("timeFormat/pattern"))
                            )
                        except ValueError as e:
                            error(e)
                    elif elem.tag == "alias":
                        time_formats = Alias(_translate_alias(["time_formats"], elem.attrib["path"]))

            datetime_formats = data.setdefault("datetime_formats", {})
            for format in calendar.findall("dateTimeFormats"):
                for elem in format.getiterator():
                    if elem.tag == "dateTimeFormatLength":
                        if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib.get(
                            "type"
                        ) in datetime_formats:
                            continue
                        try:
                            datetime_formats[elem.attrib.get("type")] = text_type(
                                elem.findtext("dateTimeFormat/pattern")
                            )
                        except ValueError as e:
                            error(e)
                    elif elem.tag == "alias":
                        datetime_formats = Alias(_translate_alias(["datetime_formats"], elem.attrib["path"]))

        # <numbers>

        number_symbols = data.setdefault("number_symbols", {})
        for elem in tree.findall(".//numbers/symbols/*"):
            if "draft" in elem.attrib or "alt" in elem.attrib:
                continue
            number_symbols[elem.tag] = text_type(elem.text)

        decimal_formats = data.setdefault("decimal_formats", {})
        for elem in tree.findall(".//decimalFormats/decimalFormatLength"):
            if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib.get("type") in decimal_formats:
                continue
            if elem.findall("./alias"):
                # TODO map the alias to its target
                continue
            pattern = text_type(elem.findtext("./decimalFormat/pattern"))
            decimal_formats[elem.attrib.get("type")] = numbers.parse_pattern(pattern)

        scientific_formats = data.setdefault("scientific_formats", {})
        for elem in tree.findall(".//scientificFormats/scientificFormatLength"):
            if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib.get("type") in scientific_formats:
                continue
            pattern = text_type(elem.findtext("scientificFormat/pattern"))
            scientific_formats[elem.attrib.get("type")] = numbers.parse_pattern(pattern)

        currency_formats = data.setdefault("currency_formats", {})
        for elem in tree.findall(".//currencyFormats/currencyFormatLength/currencyFormat"):
            if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib.get("type") in currency_formats:
                continue
            for child in elem.getiterator():
                if child.tag == "alias":
                    currency_formats[elem.attrib.get("type")] = Alias(
                        _translate_alias(["currency_formats", elem.attrib["type"]], child.attrib["path"])
                    )
                elif child.tag == "pattern":
                    pattern = text_type(child.text)
                    currency_formats[elem.attrib.get("type")] = numbers.parse_pattern(pattern)

        percent_formats = data.setdefault("percent_formats", {})
        for elem in tree.findall(".//percentFormats/percentFormatLength"):
            if ("draft" in elem.attrib or "alt" in elem.attrib) and elem.attrib.get("type") in percent_formats:
                continue
            pattern = text_type(elem.findtext("percentFormat/pattern"))
            percent_formats[elem.attrib.get("type")] = numbers.parse_pattern(pattern)

        currency_names = data.setdefault("currency_names", {})
        currency_names_plural = data.setdefault("currency_names_plural", {})
        currency_symbols = data.setdefault("currency_symbols", {})
        for elem in tree.findall(".//currencies/currency"):
            code = elem.attrib["type"]
            for name in elem.findall("displayName"):
                if ("draft" in name.attrib) and code in currency_names:
                    continue
                if "count" in name.attrib:
                    currency_names_plural.setdefault(code, {})[name.attrib["count"]] = text_type(name.text)
                else:
                    currency_names[code] = text_type(name.text)
            # TODO: support choice patterns for currency symbol selection
            symbol = elem.find("symbol")
            if symbol is not None and "draft" not in symbol.attrib and "choice" not in symbol.attrib:
                currency_symbols[code] = text_type(symbol.text)

        # <units>

        unit_patterns = data.setdefault("unit_patterns", {})
        for elem in tree.findall(".//units/unitLength"):
            unit_length_type = elem.attrib["type"]
            for unit in elem.findall("unit"):
                unit_type = unit.attrib["type"]
                for pattern in unit.findall("unitPattern"):
                    box = unit_type
                    box += ":" + unit_length_type
                    unit_patterns.setdefault(box, {})[pattern.attrib["count"]] = text_type(pattern.text)

        date_fields = data.setdefault("date_fields", {})
        for elem in tree.findall(".//dates/fields/field"):
            field_type = elem.attrib["type"]
            date_fields.setdefault(field_type, {})
            for rel_time in elem.findall("relativeTime"):
                rel_time_type = rel_time.attrib["type"]
                for pattern in rel_time.findall("relativeTimePattern"):
                    date_fields[field_type].setdefault(rel_time_type, {})[pattern.attrib["count"]] = text_type(
                        pattern.text
                    )

        outfile = open(data_filename, "wb")
        try:
            pickle.dump(data, outfile, 2)
        finally:
            outfile.close()