def __init__(self, locale=None, domain=None, header_comment=DEFAULT_HEADER, project=None, version=None, copyright_holder=None, msgid_bugs_address=None, creation_date=None, revision_date=None, last_translator=None, language_team=None, charset='utf-8', fuzzy=True): """Initialize the catalog object. :param locale: the locale identifier or `Locale` object, or `None` if the catalog is not bound to a locale (which basically means it's a template) :param domain: the message domain :param header_comment: the header comment as string, or `None` for the default header :param project: the project's name :param version: the project's version :param copyright_holder: the copyright holder of the catalog :param msgid_bugs_address: the email address or URL to submit bug reports to :param creation_date: the date the catalog was created :param revision_date: the date the catalog was revised :param last_translator: the name and email of the last translator :param language_team: the name and email of the language team :param charset: the encoding to use in the output :param fuzzy: the fuzzy bit on the catalog header """ self.domain = domain #: The message domain if locale: locale = Locale.parse(locale) self.locale = locale #: The locale or `None` self._header_comment = header_comment self._messages = odict() self.project = project or 'PROJECT' #: The project name self.version = version or 'VERSION' #: The project version self.copyright_holder = copyright_holder or 'ORGANIZATION' self.msgid_bugs_address = msgid_bugs_address or 'EMAIL@ADDRESS' self.last_translator = last_translator or 'FULL NAME <EMAIL@ADDRESS>' """Name and email address of the last translator.""" self.language_team = language_team or 'LANGUAGE <*****@*****.**>' """Name and email address of the language team.""" self.charset = charset or 'utf-8' if creation_date is None: creation_date = datetime.now(LOCALTZ) elif isinstance(creation_date, datetime) and not creation_date.tzinfo: creation_date = creation_date.replace(tzinfo=LOCALTZ) self.creation_date = creation_date #: Creation date of the template if revision_date is None: revision_date = datetime.now(LOCALTZ) elif isinstance(revision_date, datetime) and not revision_date.tzinfo: revision_date = revision_date.replace(tzinfo=LOCALTZ) self.revision_date = revision_date #: Last revision date of the catalog self.fuzzy = fuzzy #: Catalog header fuzzy bit (`True` or `False`) self.obsolete = odict() #: Dictionary of obsolete messages self._num_plurals = None self._plural_expr = None
def __init__(self, locale=None, domain=None, header_comment=DEFAULT_HEADER, project=None, version=None, copyright_holder=None, msgid_bugs_address=None, creation_date=None, revision_date=None, last_translator=None, language_team=None, charset=None, fuzzy=True): """Initialize the catalog object. :param locale: the locale identifier or `Locale` object, or `None` if the catalog is not bound to a locale (which basically means it's a template) :param domain: the message domain :param header_comment: the header comment as string, or `None` for the default header :param project: the project's name :param version: the project's version :param copyright_holder: the copyright holder of the catalog :param msgid_bugs_address: the email address or URL to submit bug reports to :param creation_date: the date the catalog was created :param revision_date: the date the catalog was revised :param last_translator: the name and email of the last translator :param language_team: the name and email of the language team :param charset: the encoding to use in the output (defaults to utf-8) :param fuzzy: the fuzzy bit on the catalog header """ self.domain = domain #: The message domain if locale: locale = Locale.parse(locale) self.locale = locale #: The locale or `None` self._header_comment = header_comment self._messages = odict() self.project = project or 'PROJECT' #: The project name self.version = version or 'VERSION' #: The project version self.copyright_holder = copyright_holder or 'ORGANIZATION' self.msgid_bugs_address = msgid_bugs_address or 'EMAIL@ADDRESS' self.last_translator = last_translator or 'FULL NAME <EMAIL@ADDRESS>' """Name and email address of the last translator.""" self.language_team = language_team or 'LANGUAGE <*****@*****.**>' """Name and email address of the language team.""" self.charset = charset or 'utf-8' if creation_date is None: creation_date = datetime.now(LOCALTZ) elif isinstance(creation_date, datetime) and not creation_date.tzinfo: creation_date = creation_date.replace(tzinfo=LOCALTZ) self.creation_date = creation_date #: Creation date of the template if revision_date is None: revision_date = 'YEAR-MO-DA HO:MI+ZONE' elif isinstance(revision_date, datetime) and not revision_date.tzinfo: revision_date = revision_date.replace(tzinfo=LOCALTZ) self.revision_date = revision_date #: Last revision date of the catalog self.fuzzy = fuzzy #: Catalog header fuzzy bit (`True` or `False`) self.obsolete = odict() #: Dictionary of obsolete messages self._num_plurals = None self._plural_expr = None
def update_using_catalog(self, catalog_to_merge, use_fuzzy_matching=False, include_obsolete=False): super(Catalog, self).update( catalog_to_merge, no_fuzzy_matching=(not use_fuzzy_matching)) # Don't use gettext's obsolete functionality as it polutes files: merge # into main translations if anything if include_obsolete: self.merge_obsolete() self.obsolete = odict()
def __init__ (self, filename, parent = None) : self.__super.__init__ () config = self._as_config_parser (filename) parent = self._as_config_parser (parent) self.patterns = odict () self.extractors = dict (python = TFL.Babel.Extractor.Python) self.defaults = dict () self._method_options = dict () for cfg in parent, config : if cfg : self._add_config (cfg)
def test_odict_pop(): odict = util.odict() odict[0] = 1 value = odict.pop(0) assert 1 == value assert [] == list(odict.items()) assert odict.pop(2, None) is None try: odict.pop(2) assert False except KeyError: assert True
def update(self, template_path=None, use_fuzzy_matching=False, include_obsolete=False, include_header=False): """Updates catalog with messages from a template.""" if template_path is None: template_path = os.path.join('translations', 'messages.pot') if not self.exists: self.init(template_path) return template_file = self.pod.open_file(template_path) template = pofile.read_po(template_file) super(Catalog, self).update( template, no_fuzzy_matching=(not use_fuzzy_matching)) # Don't use gettext's obsolete functionality as it polutes files: merge # into main translations if anything if include_obsolete: self.merge_obsolete() self.obsolete = odict() self.save(include_header=include_header)
def __init__(self, locale=None, domain=None, header_comment=DEFAULT_HEADER, project=None, version=None, copyright_holder=None, msgid_bugs_address=None, creation_date=None, revision_date=None, last_translator=None, language_team=None, charset=None, fuzzy=True, name=None, clean_msg_funcs=None, fuzzy_score_threshold=0.9): if fuzzyset is None: raise ImportError( 'Please install "fuzzyset" to use "UniqueMessagesCatalog" class' ) super(UniqueMessagesCatalog, self).__init__(locale=locale, domain=domain, header_comment=header_comment, project=project, version=version, copyright_holder=copyright_holder, msgid_bugs_address=msgid_bugs_address, creation_date=creation_date, revision_date=revision_date, last_translator=last_translator, language_team=language_team, charset=charset, fuzzy=fuzzy) self._messages_strings = FuzzySetEx() self._repeatable_messages = odict() self._clean_msg_funcs = clean_msg_funcs self._fuzzy_score_threshold = fuzzy_score_threshold self.name = name
def parse_mapping(fileobj, filename=None): """Parse an extraction method mapping from a file-like object. >>> buf = StringIO(''' ... [extractors] ... custom = mypackage.module:myfunc ... ... # Python source files ... [python: **.py] ... ... # Genshi templates ... [genshi: **/templates/**.html] ... include_attrs = ... [genshi: **/templates/**.txt] ... template_class = genshi.template:TextTemplate ... encoding = latin-1 ... ... # Some custom extractor ... [custom: **/custom/*.*] ... ''') >>> method_map, options_map = parse_mapping(buf) >>> len(method_map) 4 >>> method_map[0] ('**.py', 'python') >>> options_map['**.py'] {} >>> method_map[1] ('**/templates/**.html', 'genshi') >>> options_map['**/templates/**.html']['include_attrs'] '' >>> method_map[2] ('**/templates/**.txt', 'genshi') >>> options_map['**/templates/**.txt']['template_class'] 'genshi.template:TextTemplate' >>> options_map['**/templates/**.txt']['encoding'] 'latin-1' >>> method_map[3] ('**/custom/*.*', 'mypackage.module:myfunc') >>> options_map['**/custom/*.*'] {} :param fileobj: a readable file-like object containing the configuration text to parse :return: a `(method_map, options_map)` tuple :rtype: `tuple` :see: `extract_from_directory` """ extractors = {} method_map = [] options_map = {} parser = RawConfigParser() parser._sections = odict(parser._sections) # We need ordered sections parser.readfp(fileobj, filename) for section in parser.sections(): if section == 'extractors': extractors = dict(parser.items(section)) else: method, pattern = [part.strip() for part in section.split(':', 1)] method_map.append((pattern, method)) options_map[pattern] = dict(parser.items(section)) if extractors: for idx, (pattern, method) in enumerate(method_map): if method in extractors: method = extractors[method] method_map[idx] = (pattern, method) return (method_map, options_map)
def parse_mapping(fileobj, filename=None): """Parse an extraction method mapping from a file-like object. >>> buf = StringIO(''' ... [extractors] ... custom = mypackage.module:myfunc ... ... # Python source files ... [python: **.py] ... ... # Genshi templates ... [genshi: **/templates/**.html] ... include_attrs = ... [genshi: **/templates/**.txt] ... template_class = genshi.template:TextTemplate ... encoding = latin-1 ... ... # Some custom extractor ... [custom: **/custom/*.*] ... ''') >>> method_map, options_map = parse_mapping(buf) >>> len(method_map) 4 >>> method_map[0] ('**.py', 'python') >>> options_map['**.py'] {} >>> method_map[1] ('**/templates/**.html', 'genshi') >>> options_map['**/templates/**.html']['include_attrs'] '' >>> method_map[2] ('**/templates/**.txt', 'genshi') >>> options_map['**/templates/**.txt']['template_class'] 'genshi.template:TextTemplate' >>> options_map['**/templates/**.txt']['encoding'] 'latin-1' >>> method_map[3] ('**/custom/*.*', 'mypackage.module:myfunc') >>> options_map['**/custom/*.*'] {} :param fileobj: a readable file-like object containing the configuration text to parse :see: `extract_from_directory` """ extractors = {} method_map = [] options_map = {} parser = RawConfigParser() parser._sections = odict(parser._sections) # We need ordered sections parser.readfp(fileobj, filename) for section in parser.sections(): if section == 'extractors': extractors = dict(parser.items(section)) else: method, pattern = [part.strip() for part in section.split(':', 1)] method_map.append((pattern, method)) options_map[pattern] = dict(parser.items(section)) if extractors: for idx, (pattern, method) in enumerate(method_map): if method in extractors: method = extractors[method] method_map[idx] = (pattern, method) return (method_map, options_map)
def extract( self, include_obsolete=False, localized=False, paths=None, include_header=False, locales=None, use_fuzzy_matching=False, ): env = self.pod.create_template_env() all_locales = set(list(self.pod.list_locales())) message_ids_to_messages = {} paths_to_messages = collections.defaultdict(set) paths_to_locales = collections.defaultdict(set) comment_tags = [":"] options = {"extensions": ",".join(env.extensions.keys()), "silent": "false"} # Extract messages from content files. def callback(doc, item, key, unused_node): # Verify that the fields we're extracting are fields for a document # that's in the default locale. If not, skip the document. _handle_field(doc.pod_path, item, key, unused_node) def _handle_field(path, item, key, node): if not key.endswith("@") or not isinstance(item, basestring): return # Support gettext "extracted comments" on tagged fields. This is # consistent with extracted comments in templates, which follow # the format "{#: Extracted comment. #}". An example: # field@: Message. # field@#: Extracted comment for field@. auto_comments = [] if isinstance(node, dict): auto_comment = node.get("{}#".format(key)) if auto_comment: auto_comments.append(auto_comment) locations = [(path, 0)] existing_message = message_ids_to_messages.get(item) if existing_message: message_ids_to_messages[item].locations.extend(locations) paths_to_messages[path].add(existing_message) else: message = catalog.Message(item, None, auto_comments=auto_comments, locations=locations) message_ids_to_messages[message.id] = message paths_to_messages[path].add(message) for collection in self.pod.list_collections(): text = "Extracting collection: {}".format(collection.pod_path) self.pod.logger.info(text) for doc in collection.list_docs(include_hidden=True): if not self._should_extract(paths, doc.pod_path): continue tagged_fields = doc.get_tagged_fields() utils.walk(tagged_fields, lambda *args: callback(doc, *args)) paths_to_locales[doc.pod_path].update(doc.locales) all_locales.update(doc.locales) # Extract messages from podspec. config = self.pod.get_podspec().get_config() podspec_path = "/podspec.yaml" if self._should_extract(paths, podspec_path): self.pod.logger.info("Extracting podspec: {}".format(podspec_path)) utils.walk(config, lambda *args: _handle_field(podspec_path, *args)) # Extract messages from content and views. pod_files = [os.path.join("/views", path) for path in self.pod.list_dir("/views/")] pod_files += [os.path.join("/content", path) for path in self.pod.list_dir("/content/")] for pod_path in pod_files: if self._should_extract(paths, pod_path): pod_locales = paths_to_locales.get(pod_path) if pod_locales: text = "Extracting: {} ({} locales)" text = text.format(pod_path, len(pod_locales)) self.pod.logger.info(text) else: self.pod.logger.info("Extracting: {}".format(pod_path)) fp = self.pod.open_file(pod_path) try: all_parts = extract.extract( "jinja2.ext.babel_extract", fp, options=options, comment_tags=comment_tags ) for parts in all_parts: lineno, string, comments, context = parts locations = [(pod_path, lineno)] existing_message = message_ids_to_messages.get(string) if existing_message: message_ids_to_messages[string].locations.extend(locations) else: message = catalog.Message( string, None, auto_comments=comments, context=context, locations=locations ) paths_to_messages[pod_path].add(message) message_ids_to_messages[message.id] = message except tokenize.TokenError: self.pod.logger.error("Problem extracting: {}".format(pod_path)) raise # Localized message catalogs. if localized: for locale in all_locales: if locales and locale not in locales: continue localized_catalog = self.get(locale) if not include_obsolete: localized_catalog.obsolete = babel_util.odict() for message in list(localized_catalog): if message.id not in message_ids_to_messages: localized_catalog.delete(message.id, context=message.context) catalog_to_merge = catalog.Catalog() for path, message_items in paths_to_messages.iteritems(): locales_with_this_path = paths_to_locales.get(path) if locales_with_this_path and locale not in locales_with_this_path: continue for message in message_items: translation = None existing_message = localized_catalog.get(message.id) if existing_message: translation = existing_message.string catalog_to_merge.add( message.id, translation, locations=message.locations, auto_comments=message.auto_comments, flags=message.flags, user_comments=message.user_comments, context=message.context, lineno=message.lineno, previous_id=message.previous_id, ) localized_catalog.update_using_catalog(catalog_to_merge, use_fuzzy_matching=use_fuzzy_matching) localized_catalog.save(include_header=include_header) missing = localized_catalog.list_untranslated() num_messages = len(localized_catalog) num_translated = num_messages - len(missing) text = "Saved: /{path} ({num_translated}/{num_messages})" self.pod.logger.info( text.format( path=localized_catalog.pod_path, num_translated=num_translated, num_messages=num_messages ) ) return # Global (or missing, specified by -o) message catalog. template_path = self.template_path catalog_obj, _ = self._get_or_create_catalog(template_path) if not include_obsolete: catalog_obj.obsolete = babel_util.odict() for message in list(catalog_obj): catalog_obj.delete(message.id, context=message.context) for message in message_ids_to_messages.itervalues(): catalog_obj.add(message.id, None, locations=message.locations, auto_comments=message.auto_comments) return self.write_template( template_path, catalog_obj, include_obsolete=include_obsolete, include_header=include_header )
def update(self, template, no_fuzzy_matching=False): """Update the catalog based on the given template catalog. >>> from babel.messages import Catalog >>> template = Catalog() >>> template.add('green', locations=[('main.py', 99)]) <Message ...> >>> template.add('blue', locations=[('main.py', 100)]) <Message ...> >>> template.add(('salad', 'salads'), locations=[('util.py', 42)]) <Message ...> >>> catalog = Catalog(locale='de_DE') >>> catalog.add('blue', u'blau', locations=[('main.py', 98)]) <Message ...> >>> catalog.add('head', u'Kopf', locations=[('util.py', 33)]) <Message ...> >>> catalog.add(('salad', 'salads'), (u'Salat', u'Salate'), ... locations=[('util.py', 38)]) <Message ...> >>> catalog.update(template) >>> len(catalog) 3 >>> msg1 = catalog['green'] >>> msg1.string >>> msg1.locations [('main.py', 99)] >>> msg2 = catalog['blue'] >>> msg2.string u'blau' >>> msg2.locations [('main.py', 100)] >>> msg3 = catalog['salad'] >>> msg3.string (u'Salat', u'Salate') >>> msg3.locations [('util.py', 42)] Messages that are in the catalog but not in the template are removed from the main collection, but can still be accessed via the `obsolete` member: >>> 'head' in catalog False >>> catalog.obsolete.values() [<Message 'head' (flags: [])>] :param template: the reference catalog, usually read from a POT file :param no_fuzzy_matching: whether to use fuzzy matching of message IDs """ messages = self._messages remaining = messages.copy() self._messages = odict() # Prepare for fuzzy matching fuzzy_candidates = [] if not no_fuzzy_matching: fuzzy_candidates = dict([ (self._key_for(msgid), messages[msgid].context) for msgid in messages if msgid and messages[msgid].string ]) fuzzy_matches = set() def _merge(message, oldkey, newkey): message = message.clone() fuzzy = False if oldkey != newkey: fuzzy = True fuzzy_matches.add(oldkey) oldmsg = messages.get(oldkey) if isinstance(oldmsg.id, string_types): message.previous_id = [oldmsg.id] else: message.previous_id = list(oldmsg.id) else: oldmsg = remaining.pop(oldkey, None) message.string = oldmsg.string if isinstance(message.id, (list, tuple)): if not isinstance(message.string, (list, tuple)): fuzzy = True message.string = tuple( [message.string] + ([u''] * (len(message.id) - 1)) ) elif len(message.string) != self.num_plurals: fuzzy = True message.string = tuple(message.string[:len(oldmsg.string)]) elif isinstance(message.string, (list, tuple)): fuzzy = True message.string = message.string[0] message.flags |= oldmsg.flags if fuzzy: message.flags |= set([u'fuzzy']) self[message.id] = message for message in template: if message.id: key = self._key_for(message.id, message.context) if key in messages: _merge(message, key, key) else: if no_fuzzy_matching is False: # do some fuzzy matching with difflib if isinstance(key, tuple): matchkey = key[0] # just the msgid, no context else: matchkey = key matches = get_close_matches(matchkey.lower().strip(), fuzzy_candidates.keys(), 1) if matches: newkey = matches[0] newctxt = fuzzy_candidates[newkey] if newctxt is not None: newkey = newkey, newctxt _merge(message, newkey, key) continue self[message.id] = message for msgid in remaining: if no_fuzzy_matching or msgid not in fuzzy_matches: self.obsolete[msgid] = remaining[msgid] # Make updated catalog's POT-Creation-Date equal to the template # used to update the catalog self.creation_date = template.creation_date
def rollback_obsolete(self): self._messages.update(self.obsolete) self.obsolete = odict()
def update(self, template, no_fuzzy_matching=False): """Update the catalog based on the given template catalog. >>> from babel.messages import Catalog >>> template = Catalog() >>> template.add('green', locations=[('main.py', 99)]) >>> template.add('blue', locations=[('main.py', 100)]) >>> template.add(('salad', 'salads'), locations=[('util.py', 42)]) >>> catalog = Catalog(locale='de_DE') >>> catalog.add('blue', u'blau', locations=[('main.py', 98)]) >>> catalog.add('head', u'Kopf', locations=[('util.py', 33)]) >>> catalog.add(('salad', 'salads'), (u'Salat', u'Salate'), ... locations=[('util.py', 38)]) >>> catalog.update(template) >>> len(catalog) 3 >>> msg1 = catalog['green'] >>> msg1.string >>> msg1.locations [('main.py', 99)] >>> msg2 = catalog['blue'] >>> msg2.string u'blau' >>> msg2.locations [('main.py', 100)] >>> msg3 = catalog['salad'] >>> msg3.string (u'Salat', u'Salate') >>> msg3.locations [('util.py', 42)] Messages that are in the catalog but not in the template are removed from the main collection, but can still be accessed via the `obsolete` member: >>> 'head' in catalog False >>> catalog.obsolete.values() [<Message 'head' (flags: [])>] :param template: the reference catalog, usually read from a POT file :param no_fuzzy_matching: whether to use fuzzy matching of message IDs """ messages = self._messages remaining = messages.copy() self._messages = odict() # Prepare for fuzzy matching fuzzy_candidates = [] if not no_fuzzy_matching: fuzzy_candidates = [ self._key_for(msgid) for msgid in messages if msgid and messages[msgid].string ] fuzzy_matches = set() def _merge(message, oldkey, newkey): message = message.clone() fuzzy = False if oldkey != newkey: fuzzy = True fuzzy_matches.add(oldkey) oldmsg = messages.get(oldkey) if isinstance(oldmsg.id, basestring): message.previous_id = [oldmsg.id] else: message.previous_id = list(oldmsg.id) else: oldmsg = remaining.pop(oldkey, None) message.string = oldmsg.string if isinstance(message.id, (list, tuple)): if not isinstance(message.string, (list, tuple)): fuzzy = True message.string = tuple( [message.string] + ([u''] * (len(message.id) - 1)) ) elif len(message.string) != self.num_plurals: fuzzy = True message.string = tuple(message.string[:len(oldmsg.string)]) elif isinstance(message.string, (list, tuple)): fuzzy = True message.string = message.string[0] message.flags |= oldmsg.flags if fuzzy: message.flags |= set([u'fuzzy']) self[message.id] = message for message in template: if message.id: key = self._key_for(message.id) if key in messages: _merge(message, key, key) else: if no_fuzzy_matching is False: # do some fuzzy matching with difflib matches = get_close_matches(key.lower().strip(), fuzzy_candidates, 1) if matches: _merge(message, matches[0], key) continue self[message.id] = message self.obsolete = odict() for msgid in remaining: if no_fuzzy_matching or msgid not in fuzzy_matches: self.obsolete[msgid] = remaining[msgid]
def _rollback_obsolete(catalog): catalog._messages.update(catalog.obsolete) catalog.obsolete = odict()
def update(self, template, no_fuzzy_matching=False): """Update the catalog based on the given template catalog. >>> from babel.messages import Catalog >>> template = Catalog() >>> template.add('green', locations=[('main.py', 99)]) <Message ...> >>> template.add('blue', locations=[('main.py', 100)]) <Message ...> >>> template.add(('salad', 'salads'), locations=[('util.py', 42)]) <Message ...> >>> catalog = Catalog(locale='de_DE') >>> catalog.add('blue', u('blau'), locations=[('main.py', 98)]) <Message ...> >>> catalog.add('head', u('Kopf'), locations=[('util.py', 33)]) <Message ...> >>> catalog.add(('salad', 'salads'), (u('Salat'), u('Salate')), ... locations=[('util.py', 38)]) <Message ...> >>> catalog.update(template) >>> len(catalog) 3 >>> msg1 = catalog['green'] >>> msg1.string >>> msg1.locations [('main.py', 99)] >>> msg2 = catalog['blue'] >>> print(msg2.string) blau >>> msg2.locations [('main.py', 100)] >>> msg3 = catalog['salad'] >>> print(msg3.string[0]) Salat >>> print(msg3.string[1]) Salate >>> msg3.locations [('util.py', 42)] Messages that are in the catalog but not in the template are removed from the main collection, but can still be accessed via the `obsolete` member: >>> 'head' in catalog False >>> for v in catalog.obsolete.values(): ... print(v) <Message head (flags: [])> :param template: the reference catalog, usually read from a POT file :param no_fuzzy_matching: whether to use fuzzy matching of message IDs """ messages = self._messages remaining = messages.copy() self._messages = odict() # Prepare for fuzzy matching fuzzy_candidates = [] if not no_fuzzy_matching: fuzzy_candidates = dict([ (self._key_for(msgid), messages[msgid].context) for msgid in messages if msgid and messages[msgid].string ]) fuzzy_matches = set() def _merge(message, oldkey, newkey): message = message.clone() fuzzy = False if oldkey != newkey: fuzzy = True fuzzy_matches.add(oldkey) oldmsg = messages.get(oldkey) if isinstance(oldmsg.id, string_types): message.previous_id = [oldmsg.id] else: message.previous_id = list(oldmsg.id) else: oldmsg = remaining.pop(oldkey, None) message.string = oldmsg.string if isinstance(message.id, (list, tuple)): if not isinstance(message.string, (list, tuple)): fuzzy = True message.string = tuple([message.string] + ([u('')] * (len(message.id) - 1))) elif len(message.string) != self.num_plurals: fuzzy = True message.string = tuple(message.string[:len(oldmsg.string)]) elif isinstance(message.string, (list, tuple)): fuzzy = True message.string = message.string[0] message.flags |= oldmsg.flags if fuzzy: message.flags |= set([u('fuzzy')]) self[message.id] = message for message in template: if message.id: key = self._key_for(message.id, message.context) if key in messages: _merge(message, key, key) else: if no_fuzzy_matching is False: # do some fuzzy matching with difflib if isinstance(key, tuple): matchkey = key[0] # just the msgid, no context else: matchkey = key matches = get_close_matches(matchkey.lower().strip(), fuzzy_candidates.keys(), 1) if matches: newkey = matches[0] newctxt = fuzzy_candidates[newkey] if newctxt is not None: newkey = newkey, newctxt _merge(message, newkey, key) continue self[message.id] = message self.obsolete = odict() for msgid in remaining: if no_fuzzy_matching or msgid not in fuzzy_matches: self.obsolete[msgid] = remaining[msgid] # Make updated catalog's POT-Creation-Date equal to the template # used to update the catalog self.creation_date = template.creation_date
def extract(self, include_obsolete=False, localized=False, paths=None, include_header=False, locales=None, use_fuzzy_matching=False): env = self.pod.get_jinja_env() all_locales = set(list(self.pod.list_locales())) message_ids_to_messages = {} paths_to_messages = collections.defaultdict(set) paths_to_locales = collections.defaultdict(set) comment_tags = [ ':', ] options = { 'extensions': ','.join(env.extensions.keys()), 'silent': 'false', } # Extract from content files. def callback(doc, item, key, unused_node): # Verify that the fields we're extracting are fields for a document # that's in the default locale. If not, skip the document. _handle_field(doc.pod_path, item, key, unused_node) def _add_existing_message(msgid, locations, auto_comments=None, context=None, path=None): existing_message = message_ids_to_messages.get(msgid) auto_comments = [] if auto_comments is None else auto_comments if existing_message: message_ids_to_messages[msgid].locations.extend(locations) paths_to_messages[path].add(existing_message) else: message = catalog.Message( msgid, None, auto_comments=auto_comments, context=context, locations=locations) paths_to_messages[path].add(message) message_ids_to_messages[message.id] = message def _handle_field(path, item, key, node): if (not key or not isinstance(item, basestring) or not isinstance(key, basestring) or not key.endswith('@')): return # Support gettext "extracted comments" on tagged fields. This is # consistent with extracted comments in templates, which follow # the format "{#: Extracted comment. #}". An example: # field@: Message. # field@#: Extracted comment for field@. auto_comments = [] if isinstance(node, dict): auto_comment = node.get('{}#'.format(key)) if auto_comment: auto_comments.append(auto_comment) locations = [(path, 0)] _add_existing_message( msgid=item, auto_comments=auto_comments, locations=locations, path=path) for collection in self.pod.list_collections(): text = 'Extracting collection: {}'.format(collection.pod_path) self.pod.logger.info(text) # Extract from blueprint. utils.walk(collection.tagged_fields, lambda *args: callback(collection, *args)) # Extract from docs in collection. for doc in collection.docs(include_hidden=True): if not self._should_extract_as_babel(paths, doc.pod_path): continue tagged_fields = doc.get_tagged_fields() utils.walk(tagged_fields, lambda *args: callback(doc, *args)) paths_to_locales[doc.pod_path].update(doc.locales) all_locales.update(doc.locales) # Extract from podspec. config = self.pod.get_podspec().get_config() podspec_path = '/podspec.yaml' if self._should_extract_as_babel(paths, podspec_path): self.pod.logger.info('Extracting podspec: {}'.format(podspec_path)) utils.walk(config, lambda *args: _handle_field(podspec_path, *args)) # Extract from content and views. pod_files = [os.path.join('/views', path) for path in self.pod.list_dir('/views/')] pod_files += [os.path.join('/content', path) for path in self.pod.list_dir('/content/')] pod_files += [os.path.join('/data', path) for path in self.pod.list_dir('/data/')] for pod_path in pod_files: if self._should_extract_as_csv(paths, pod_path): rows = utils.get_rows_from_csv(self.pod, pod_path) self.pod.logger.info('Extracting: {}'.format(pod_path)) for row in rows: for i, parts in enumerate(row.iteritems()): key, val = parts if key.endswith('@'): locations = [(pod_path, i)] _add_existing_message( msgid=val, locations=locations, path=pod_path) elif self._should_extract_as_babel(paths, pod_path): if pod_path.startswith('/data') and pod_path.endswith(('.yaml', '.yml')): self.pod.logger.info('Extracting: {}'.format(pod_path)) content = self.pod.read_file(pod_path) fields = utils.load_yaml(content, pod=self.pod) utils.walk(fields, lambda *args: _handle_field(pod_path, *args)) continue pod_locales = paths_to_locales.get(pod_path) if pod_locales: text = 'Extracting: {} ({} locales)' text = text.format(pod_path, len(pod_locales)) self.pod.logger.info(text) else: self.pod.logger.info('Extracting: {}'.format(pod_path)) fp = self.pod.open_file(pod_path) try: all_parts = extract.extract( 'jinja2.ext.babel_extract', fp, options=options, comment_tags=comment_tags) for parts in all_parts: lineno, string, comments, context = parts locations = [(pod_path, lineno)] _add_existing_message( msgid=string, auto_comments=comments, context=context, locations=locations, path=pod_path) except tokenize.TokenError: self.pod.logger.error('Problem extracting: {}'.format(pod_path)) raise # Localized message catalogs. if localized: for locale in all_locales: if locales and locale not in locales: continue localized_catalog = self.get(locale) if not include_obsolete: localized_catalog.obsolete = babel_util.odict() for message in list(localized_catalog): if message.id not in message_ids_to_messages: localized_catalog.delete(message.id, context=message.context) catalog_to_merge = catalog.Catalog() for path, message_items in paths_to_messages.iteritems(): locales_with_this_path = paths_to_locales.get(path) if locales_with_this_path and locale not in locales_with_this_path: continue for message in message_items: translation = None existing_message = localized_catalog.get(message.id) if existing_message: translation = existing_message.string catalog_to_merge.add( message.id, translation, locations=message.locations, auto_comments=message.auto_comments, flags=message.flags, user_comments=message.user_comments, context=message.context, lineno=message.lineno, previous_id=message.previous_id) localized_catalog.update_using_catalog( catalog_to_merge, use_fuzzy_matching=use_fuzzy_matching) localized_catalog.save(include_header=include_header) missing = localized_catalog.list_untranslated() num_messages = len(localized_catalog) num_translated = num_messages - len(missing) text = 'Saved: /{path} ({num_translated}/{num_messages})' self.pod.logger.info( text.format(path=localized_catalog.pod_path, num_translated=num_translated, num_messages=num_messages)) return # Global (or missing, specified by -o) message catalog. template_path = self.template_path catalog_obj, _ = self._get_or_create_catalog(template_path) if not include_obsolete: catalog_obj.obsolete = babel_util.odict() for message in list(catalog_obj): catalog_obj.delete(message.id, context=message.context) for message in message_ids_to_messages.itervalues(): if message.id: catalog_obj.add(message.id, None, locations=message.locations, auto_comments=message.auto_comments) return self.write_template( template_path, catalog_obj, include_obsolete=include_obsolete, include_header=include_header)