def importGeckoLocales(params, files): import localeTools # FIXME: localeTools doesn't use real Chrome locales, it uses dash as # separator instead. convert_locale_code = lambda code: code.replace('-', '_') # We need to map Chrome locales to Gecko locales. Start by mapping Chrome # locales to themselves, merely with the dash as separator. locale_mapping = {convert_locale_code(l): l for l in localeTools.chromeLocales} # Convert values to Crowdin locales first (use Chrome => Crowdin mapping). for chrome_locale, crowdin_locale in localeTools.langMappingChrome.iteritems(): locale_mapping[convert_locale_code(chrome_locale)] = crowdin_locale # Now convert values to Gecko locales (use Gecko => Crowdin mapping). reverse_mapping = {v: k for k, v in locale_mapping.iteritems()} for gecko_locale, crowdin_locale in localeTools.langMappingGecko.iteritems(): if crowdin_locale in reverse_mapping: locale_mapping[reverse_mapping[crowdin_locale]] = gecko_locale for target, source in locale_mapping.iteritems(): targetFile = '_locales/%s/messages.json' % target if not targetFile in files: continue for item in params['metadata'].items('import_locales'): fileName, keys = item parts = map(lambda n: source if n == '*' else n, fileName.split('/')) sourceFile = os.path.join(os.path.dirname(item.source), *parts) incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incomplete') if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker): continue data = json.loads(files[targetFile].decode('utf-8')) try: if sourceFile.endswith('.json'): with io.open(sourceFile, 'r', encoding='utf-8') as handle: sourceData = {k: v['message'] for k, v in json.load(handle).iteritems()} else: sourceData = localeTools.readFile(sourceFile) # Resolve wildcard imports if keys == '*' or keys == '=*': importList = sourceData.keys() importList = filter(lambda k: not k.startswith('_'), importList) if keys == '=*': importList = map(lambda k: '=' + k, importList) keys = ' '.join(importList) for stringID in keys.split(): noMangling = False if stringID.startswith('='): stringID = stringID[1:] noMangling = True if stringID in sourceData: if noMangling: key = re.sub(r'\W', '_', stringID) else: key = re.sub(r'\..*', '', parts[-1]) + '_' + re.sub(r'\W', '_', stringID) if key in data: print 'Warning: locale string %s defined multiple times' % key # Remove access keys value = sourceData[stringID] match = re.search(r'^(.*?)\s*\(&.\)$', value) if match: value = match.group(1) else: index = value.find("&") if index >= 0: value = value[0:index] + value[index + 1:] data[key] = {'message': value} except Exception, e: print 'Warning: error importing locale data from %s: %s' % (sourceFile, e) files[targetFile] = toJson(data)
def importGeckoLocales(params, files): import localeTools localeCodeMapping = { 'ar': 'ar', 'bg': 'bg', 'ca': 'ca', 'cs': 'cs', 'da': 'da', 'de': 'de', 'el': 'el', 'en-US': 'en_US', 'en-GB': 'en_GB', 'es-ES': 'es', 'es-AR': 'es_419', 'et': 'et', 'fi': 'fi', # '': 'fil', ??? 'fr': 'fr', 'he': 'he', 'hi-IN': 'hi', 'hr': 'hr', 'hu': 'hu', 'id': 'id', 'it': 'it', 'ja': 'ja', 'ko': 'ko', 'lt': 'lt', 'lv': 'lv', 'nl': 'nl', # 'nb-NO': 'no', ??? 'pl': 'pl', 'pt-BR': 'pt_BR', 'pt-PT': 'pt_PT', 'ro': 'ro', 'ru': 'ru', 'sk': 'sk', 'sl': 'sl', 'sr': 'sr', 'sv-SE': 'sv', 'th': 'th', 'tr': 'tr', 'uk': 'uk', 'vi': 'vi', 'zh-CN': 'zh_CN', 'zh-TW': 'zh_TW', } for source, target in localeCodeMapping.iteritems(): targetFile = '_locales/%s/messages.json' % target for item in params['metadata'].items('import_locales'): fileName, keys = item parts = map(lambda n: source if n == '*' else n, fileName.split('/')) sourceFile = os.path.join(os.path.dirname(item.source), *parts) incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incomplete') if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker): continue data = {} if targetFile in files: data = json.loads(files[targetFile].decode('utf-8')) try: sourceData = localeTools.readFile(sourceFile) # Resolve wildcard imports if keys == '*' or keys == '=*': importList = sourceData.keys() importList = filter(lambda k: not k.startswith('_'), importList) if keys == '=*': importList = map(lambda k: '=' + k, importList) keys = ' '.join(importList) for stringID in re.split(r'\s+', keys): noMangling = False if stringID.startswith('='): stringID = stringID[1:] noMangling = True if stringID in sourceData: if noMangling: key = re.sub(r'\W', '_', stringID) else: key = re.sub(r'\..*', '', parts[-1]) + '_' + re.sub(r'\W', '_', stringID) if key in data: print 'Warning: locale string %s defined multiple times' % key # Remove access keys value = sourceData[stringID] match = re.search(r'^(.*?)\s*\(&.\)$', value) if match: value = match.group(1) else: index = value.find("&") if index >= 0: value = value[0:index] + value[index + 1:] data[key] = {'message': value} except Exception, e: print 'Warning: error importing locale data from %s: %s' % (sourceFile, e) files[targetFile] = json.dumps(data, ensure_ascii=False, sort_keys=True, indent=2, separators=(',', ': ')).encode('utf-8') + '\n'
def importGeckoLocales(params, files): import localeTools # FIXME: localeTools doesn't use real Chrome locales, it uses dash as # separator instead. convert_locale_code = lambda code: code.replace('-', '_') # We need to map Chrome locales to Gecko locales. Start by mapping Chrome # locales to themselves, merely with the dash as separator. locale_mapping = { convert_locale_code(l): l for l in localeTools.chromeLocales } # Convert values to Crowdin locales first (use Chrome => Crowdin mapping). for chrome_locale, crowdin_locale in localeTools.langMappingChrome.iteritems( ): locale_mapping[convert_locale_code(chrome_locale)] = crowdin_locale # Now convert values to Gecko locales (use Gecko => Crowdin mapping). reverse_mapping = {v: k for k, v in locale_mapping.iteritems()} for gecko_locale, crowdin_locale in localeTools.langMappingGecko.iteritems( ): if crowdin_locale in reverse_mapping: locale_mapping[reverse_mapping[crowdin_locale]] = gecko_locale for target, source in locale_mapping.iteritems(): targetFile = '_locales/%s/messages.json' % target if not targetFile in files: continue for item in params['metadata'].items('import_locales'): fileName, keys = item parts = map(lambda n: source if n == '*' else n, fileName.split('/')) sourceFile = os.path.join(os.path.dirname(item.source), *parts) incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incomplete') if not os.path.exists(sourceFile) or os.path.exists( incompleteMarker): continue data = json.loads(files[targetFile].decode('utf-8')) try: if sourceFile.endswith('.json'): with io.open(sourceFile, 'r', encoding='utf-8') as handle: sourceData = { k: v['message'] for k, v in json.load(handle).iteritems() } else: sourceData = localeTools.readFile(sourceFile) # Resolve wildcard imports if keys == '*' or keys == '=*': importList = sourceData.keys() importList = filter(lambda k: not k.startswith('_'), importList) if keys == '=*': importList = map(lambda k: '=' + k, importList) keys = ' '.join(importList) for stringID in keys.split(): noMangling = False if stringID.startswith('='): stringID = stringID[1:] noMangling = True if stringID in sourceData: if noMangling: key = re.sub(r'\W', '_', stringID) else: key = re.sub(r'\..*', '', parts[-1]) + '_' + re.sub( r'\W', '_', stringID) if key in data: print 'Warning: locale string %s defined multiple times' % key # Remove access keys value = sourceData[stringID] match = re.search(r'^(.*?)\s*\(&.\)$', value) if match: value = match.group(1) else: index = value.find('&') if index >= 0: value = value[0:index] + value[index + 1:] data[key] = {'message': value} except Exception as e: print 'Warning: error importing locale data from %s: %s' % ( sourceFile, e) files[targetFile] = toJson(data)