def __init__(self, locale=None, domain=None, header_comment=DEFAULT_HEADER,
project=None, version=None, copyright_holder=None,
msgid_bugs_address=None, creation_date=None,
revision_date=None, last_translator=None, language_team=None,
charset='utf-8', fuzzy=True):
"""Initialize the catalog object.
:param locale: the locale identifier or `Locale` object, or `None`
if the catalog is not bound to a locale (which basically
means it's a template)
:param domain: the message domain
:param header_comment: the header comment as string, or `None` for the
default header
:param project: the project's name
:param version: the project's version
:param copyright_holder: the copyright holder of the catalog
:param msgid_bugs_address: the email address or URL to submit bug
reports to
:param creation_date: the date the catalog was created
:param revision_date: the date the catalog was revised
:param last_translator: the name and email of the last translator
:param language_team: the name and email of the language team
:param charset: the encoding to use in the output
:param fuzzy: the fuzzy bit on the catalog header
"""
self.domain = domain #: The message domain
if locale:
locale = Locale.parse(locale)
self.locale = locale #: The locale or `None`
self._header_comment = header_comment
self._messages = odict()
self.project = project or 'PROJECT' #: The project name
self.version = version or 'VERSION' #: The project version
self.copyright_holder = copyright_holder or 'ORGANIZATION'
self.msgid_bugs_address = msgid_bugs_address or 'EMAIL@ADDRESS'
self.last_translator = last_translator or 'FULL NAME <EMAIL@ADDRESS>'
"""Name and email address of the last translator."""
self.language_team = language_team or 'LANGUAGE <*****@*****.**>'
"""Name and email address of the language team."""
self.charset = charset or 'utf-8'
if creation_date is None:
creation_date = datetime.now(LOCALTZ)
elif isinstance(creation_date, datetime) and not creation_date.tzinfo:
creation_date = creation_date.replace(tzinfo=LOCALTZ)
self.creation_date = creation_date #: Creation date of the template
if revision_date is None:
revision_date = datetime.now(LOCALTZ)
elif isinstance(revision_date, datetime) and not revision_date.tzinfo:
revision_date = revision_date.replace(tzinfo=LOCALTZ)
self.revision_date = revision_date #: Last revision date of the catalog
self.fuzzy = fuzzy #: Catalog header fuzzy bit (`True` or `False`)
self.obsolete = odict() #: Dictionary of obsolete messages
self._num_plurals = None
self._plural_expr = None
def __init__(self, locale=None, domain=None, header_comment=DEFAULT_HEADER,
project=None, version=None, copyright_holder=None,
msgid_bugs_address=None, creation_date=None,
revision_date=None, last_translator=None, language_team=None,
charset=None, fuzzy=True):
"""Initialize the catalog object.
:param locale: the locale identifier or `Locale` object, or `None`
if the catalog is not bound to a locale (which basically
means it's a template)
:param domain: the message domain
:param header_comment: the header comment as string, or `None` for the
default header
:param project: the project's name
:param version: the project's version
:param copyright_holder: the copyright holder of the catalog
:param msgid_bugs_address: the email address or URL to submit bug
reports to
:param creation_date: the date the catalog was created
:param revision_date: the date the catalog was revised
:param last_translator: the name and email of the last translator
:param language_team: the name and email of the language team
:param charset: the encoding to use in the output (defaults to utf-8)
:param fuzzy: the fuzzy bit on the catalog header
"""
self.domain = domain #: The message domain
if locale:
locale = Locale.parse(locale)
self.locale = locale #: The locale or `None`
self._header_comment = header_comment
self._messages = odict()
self.project = project or 'PROJECT' #: The project name
self.version = version or 'VERSION' #: The project version
self.copyright_holder = copyright_holder or 'ORGANIZATION'
self.msgid_bugs_address = msgid_bugs_address or 'EMAIL@ADDRESS'
self.last_translator = last_translator or 'FULL NAME <EMAIL@ADDRESS>'
"""Name and email address of the last translator."""
self.language_team = language_team or 'LANGUAGE <*****@*****.**>'
"""Name and email address of the language team."""
self.charset = charset or 'utf-8'
if creation_date is None:
creation_date = datetime.now(LOCALTZ)
elif isinstance(creation_date, datetime) and not creation_date.tzinfo:
creation_date = creation_date.replace(tzinfo=LOCALTZ)
self.creation_date = creation_date #: Creation date of the template
if revision_date is None:
revision_date = 'YEAR-MO-DA HO:MI+ZONE'
elif isinstance(revision_date, datetime) and not revision_date.tzinfo:
revision_date = revision_date.replace(tzinfo=LOCALTZ)
self.revision_date = revision_date #: Last revision date of the catalog
self.fuzzy = fuzzy #: Catalog header fuzzy bit (`True` or `False`)
self.obsolete = odict() #: Dictionary of obsolete messages
self._num_plurals = None
self._plural_expr = None
def update_using_catalog(self, catalog_to_merge, use_fuzzy_matching=False,
include_obsolete=False):
super(Catalog, self).update(
catalog_to_merge, no_fuzzy_matching=(not use_fuzzy_matching))
# Don't use gettext's obsolete functionality as it polutes files: merge
# into main translations if anything
if include_obsolete:
self.merge_obsolete()
self.obsolete = odict()
def update_using_catalog(self, catalog_to_merge, use_fuzzy_matching=False,
include_obsolete=False):
super(Catalog, self).update(
catalog_to_merge, no_fuzzy_matching=(not use_fuzzy_matching))
# Don't use gettext's obsolete functionality as it polutes files: merge
# into main translations if anything
if include_obsolete:
self.merge_obsolete()
self.obsolete = odict()
def __init__ (self, filename, parent = None) :
self.__super.__init__ ()
config = self._as_config_parser (filename)
parent = self._as_config_parser (parent)
self.patterns = odict ()
self.extractors = dict (python = TFL.Babel.Extractor.Python)
self.defaults = dict ()
self._method_options = dict ()
for cfg in parent, config :
if cfg :
self._add_config (cfg)
def __init__ (self, filename, parent = None) :
self.__super.__init__ ()
config = self._as_config_parser (filename)
parent = self._as_config_parser (parent)
self.patterns = odict ()
self.extractors = dict (python = TFL.Babel.Extractor.Python)
self.defaults = dict ()
self._method_options = dict ()
for cfg in parent, config :
if cfg :
self._add_config (cfg)
def test_odict_pop():
odict = util.odict()
odict[0] = 1
value = odict.pop(0)
assert 1 == value
assert [] == list(odict.items())
assert odict.pop(2, None) is None
try:
odict.pop(2)
assert False
except KeyError:
assert True
def test_odict_pop():
odict = util.odict()
odict[0] = 1
value = odict.pop(0)
assert 1 == value
assert [] == list(odict.items())
assert odict.pop(2, None) is None
try:
odict.pop(2)
assert False
except KeyError:
assert True
def update(self, template_path=None, use_fuzzy_matching=False,
include_obsolete=False, include_header=False):
"""Updates catalog with messages from a template."""
if template_path is None:
template_path = os.path.join('translations', 'messages.pot')
if not self.exists:
self.init(template_path)
return
template_file = self.pod.open_file(template_path)
template = pofile.read_po(template_file)
super(Catalog, self).update(
template, no_fuzzy_matching=(not use_fuzzy_matching))
# Don't use gettext's obsolete functionality as it polutes files: merge
# into main translations if anything
if include_obsolete:
self.merge_obsolete()
self.obsolete = odict()
self.save(include_header=include_header)
def update(self, template_path=None, use_fuzzy_matching=False,
include_obsolete=False, include_header=False):
"""Updates catalog with messages from a template."""
if template_path is None:
template_path = os.path.join('translations', 'messages.pot')
if not self.exists:
self.init(template_path)
return
template_file = self.pod.open_file(template_path)
template = pofile.read_po(template_file)
super(Catalog, self).update(
template, no_fuzzy_matching=(not use_fuzzy_matching))
# Don't use gettext's obsolete functionality as it polutes files: merge
# into main translations if anything
if include_obsolete:
self.merge_obsolete()
self.obsolete = odict()
self.save(include_header=include_header)
def __init__(self,
locale=None,
domain=None,
header_comment=DEFAULT_HEADER,
project=None,
version=None,
copyright_holder=None,
msgid_bugs_address=None,
creation_date=None,
revision_date=None,
last_translator=None,
language_team=None,
charset=None,
fuzzy=True,
name=None,
clean_msg_funcs=None,
fuzzy_score_threshold=0.9):
if fuzzyset is None:
raise ImportError(
'Please install "fuzzyset" to use "UniqueMessagesCatalog" class'
)
super(UniqueMessagesCatalog,
self).__init__(locale=locale,
domain=domain,
header_comment=header_comment,
project=project,
version=version,
copyright_holder=copyright_holder,
msgid_bugs_address=msgid_bugs_address,
creation_date=creation_date,
revision_date=revision_date,
last_translator=last_translator,
language_team=language_team,
charset=charset,
fuzzy=fuzzy)
self._messages_strings = FuzzySetEx()
self._repeatable_messages = odict()
self._clean_msg_funcs = clean_msg_funcs
self._fuzzy_score_threshold = fuzzy_score_threshold
self.name = name
def parse_mapping(fileobj, filename=None):
"""Parse an extraction method mapping from a file-like object.
>>> buf = StringIO('''
... [extractors]
... custom = mypackage.module:myfunc
...
... # Python source files
... [python: **.py]
...
... # Genshi templates
... [genshi: **/templates/**.html]
... include_attrs =
... [genshi: **/templates/**.txt]
... template_class = genshi.template:TextTemplate
... encoding = latin-1
...
... # Some custom extractor
... [custom: **/custom/*.*]
... ''')
>>> method_map, options_map = parse_mapping(buf)
>>> len(method_map)
4
>>> method_map[0]
('**.py', 'python')
>>> options_map['**.py']
{}
>>> method_map[1]
('**/templates/**.html', 'genshi')
>>> options_map['**/templates/**.html']['include_attrs']
''
>>> method_map[2]
('**/templates/**.txt', 'genshi')
>>> options_map['**/templates/**.txt']['template_class']
'genshi.template:TextTemplate'
>>> options_map['**/templates/**.txt']['encoding']
'latin-1'
>>> method_map[3]
('**/custom/*.*', 'mypackage.module:myfunc')
>>> options_map['**/custom/*.*']
{}
:param fileobj: a readable file-like object containing the configuration
text to parse
:return: a `(method_map, options_map)` tuple
:rtype: `tuple`
:see: `extract_from_directory`
"""
extractors = {}
method_map = []
options_map = {}
parser = RawConfigParser()
parser._sections = odict(parser._sections) # We need ordered sections
parser.readfp(fileobj, filename)
for section in parser.sections():
if section == 'extractors':
extractors = dict(parser.items(section))
else:
method, pattern = [part.strip() for part in section.split(':', 1)]
method_map.append((pattern, method))
options_map[pattern] = dict(parser.items(section))
if extractors:
for idx, (pattern, method) in enumerate(method_map):
if method in extractors:
method = extractors[method]
method_map[idx] = (pattern, method)
return (method_map, options_map)
def parse_mapping(fileobj, filename=None):
"""Parse an extraction method mapping from a file-like object.
>>> buf = StringIO('''
... [extractors]
... custom = mypackage.module:myfunc
...
... # Python source files
... [python: **.py]
...
... # Genshi templates
... [genshi: **/templates/**.html]
... include_attrs =
... [genshi: **/templates/**.txt]
... template_class = genshi.template:TextTemplate
... encoding = latin-1
...
... # Some custom extractor
... [custom: **/custom/*.*]
... ''')
>>> method_map, options_map = parse_mapping(buf)
>>> len(method_map)
4
>>> method_map[0]
('**.py', 'python')
>>> options_map['**.py']
{}
>>> method_map[1]
('**/templates/**.html', 'genshi')
>>> options_map['**/templates/**.html']['include_attrs']
''
>>> method_map[2]
('**/templates/**.txt', 'genshi')
>>> options_map['**/templates/**.txt']['template_class']
'genshi.template:TextTemplate'
>>> options_map['**/templates/**.txt']['encoding']
'latin-1'
>>> method_map[3]
('**/custom/*.*', 'mypackage.module:myfunc')
>>> options_map['**/custom/*.*']
{}
:param fileobj: a readable file-like object containing the configuration
text to parse
:see: `extract_from_directory`
"""
extractors = {}
method_map = []
options_map = {}
parser = RawConfigParser()
parser._sections = odict(parser._sections) # We need ordered sections
parser.readfp(fileobj, filename)
for section in parser.sections():
if section == 'extractors':
extractors = dict(parser.items(section))
else:
method, pattern = [part.strip() for part in section.split(':', 1)]
method_map.append((pattern, method))
options_map[pattern] = dict(parser.items(section))
if extractors:
for idx, (pattern, method) in enumerate(method_map):
if method in extractors:
method = extractors[method]
method_map[idx] = (pattern, method)
return (method_map, options_map)
def extract(
self,
include_obsolete=False,
localized=False,
paths=None,
include_header=False,
locales=None,
use_fuzzy_matching=False,
):
env = self.pod.create_template_env()
all_locales = set(list(self.pod.list_locales()))
message_ids_to_messages = {}
paths_to_messages = collections.defaultdict(set)
paths_to_locales = collections.defaultdict(set)
comment_tags = [":"]
options = {"extensions": ",".join(env.extensions.keys()), "silent": "false"}
# Extract messages from content files.
def callback(doc, item, key, unused_node):
# Verify that the fields we're extracting are fields for a document
# that's in the default locale. If not, skip the document.
_handle_field(doc.pod_path, item, key, unused_node)
def _handle_field(path, item, key, node):
if not key.endswith("@") or not isinstance(item, basestring):
return
# Support gettext "extracted comments" on tagged fields. This is
# consistent with extracted comments in templates, which follow
# the format "{#: Extracted comment. #}". An example:
# field@: Message.
# field@#: Extracted comment for field@.
auto_comments = []
if isinstance(node, dict):
auto_comment = node.get("{}#".format(key))
if auto_comment:
auto_comments.append(auto_comment)
locations = [(path, 0)]
existing_message = message_ids_to_messages.get(item)
if existing_message:
message_ids_to_messages[item].locations.extend(locations)
paths_to_messages[path].add(existing_message)
else:
message = catalog.Message(item, None, auto_comments=auto_comments, locations=locations)
message_ids_to_messages[message.id] = message
paths_to_messages[path].add(message)
for collection in self.pod.list_collections():
text = "Extracting collection: {}".format(collection.pod_path)
self.pod.logger.info(text)
for doc in collection.list_docs(include_hidden=True):
if not self._should_extract(paths, doc.pod_path):
continue
tagged_fields = doc.get_tagged_fields()
utils.walk(tagged_fields, lambda *args: callback(doc, *args))
paths_to_locales[doc.pod_path].update(doc.locales)
all_locales.update(doc.locales)
# Extract messages from podspec.
config = self.pod.get_podspec().get_config()
podspec_path = "/podspec.yaml"
if self._should_extract(paths, podspec_path):
self.pod.logger.info("Extracting podspec: {}".format(podspec_path))
utils.walk(config, lambda *args: _handle_field(podspec_path, *args))
# Extract messages from content and views.
pod_files = [os.path.join("/views", path) for path in self.pod.list_dir("/views/")]
pod_files += [os.path.join("/content", path) for path in self.pod.list_dir("/content/")]
for pod_path in pod_files:
if self._should_extract(paths, pod_path):
pod_locales = paths_to_locales.get(pod_path)
if pod_locales:
text = "Extracting: {} ({} locales)"
text = text.format(pod_path, len(pod_locales))
self.pod.logger.info(text)
else:
self.pod.logger.info("Extracting: {}".format(pod_path))
fp = self.pod.open_file(pod_path)
try:
all_parts = extract.extract(
"jinja2.ext.babel_extract", fp, options=options, comment_tags=comment_tags
)
for parts in all_parts:
lineno, string, comments, context = parts
locations = [(pod_path, lineno)]
existing_message = message_ids_to_messages.get(string)
if existing_message:
message_ids_to_messages[string].locations.extend(locations)
else:
message = catalog.Message(
string, None, auto_comments=comments, context=context, locations=locations
)
paths_to_messages[pod_path].add(message)
message_ids_to_messages[message.id] = message
except tokenize.TokenError:
self.pod.logger.error("Problem extracting: {}".format(pod_path))
raise
# Localized message catalogs.
if localized:
for locale in all_locales:
if locales and locale not in locales:
continue
localized_catalog = self.get(locale)
if not include_obsolete:
localized_catalog.obsolete = babel_util.odict()
for message in list(localized_catalog):
if message.id not in message_ids_to_messages:
localized_catalog.delete(message.id, context=message.context)
catalog_to_merge = catalog.Catalog()
for path, message_items in paths_to_messages.iteritems():
locales_with_this_path = paths_to_locales.get(path)
if locales_with_this_path and locale not in locales_with_this_path:
continue
for message in message_items:
translation = None
existing_message = localized_catalog.get(message.id)
if existing_message:
translation = existing_message.string
catalog_to_merge.add(
message.id,
translation,
locations=message.locations,
auto_comments=message.auto_comments,
flags=message.flags,
user_comments=message.user_comments,
context=message.context,
lineno=message.lineno,
previous_id=message.previous_id,
)
localized_catalog.update_using_catalog(catalog_to_merge, use_fuzzy_matching=use_fuzzy_matching)
localized_catalog.save(include_header=include_header)
missing = localized_catalog.list_untranslated()
num_messages = len(localized_catalog)
num_translated = num_messages - len(missing)
text = "Saved: /{path} ({num_translated}/{num_messages})"
self.pod.logger.info(
text.format(
path=localized_catalog.pod_path, num_translated=num_translated, num_messages=num_messages
)
)
return
# Global (or missing, specified by -o) message catalog.
template_path = self.template_path
catalog_obj, _ = self._get_or_create_catalog(template_path)
if not include_obsolete:
catalog_obj.obsolete = babel_util.odict()
for message in list(catalog_obj):
catalog_obj.delete(message.id, context=message.context)
for message in message_ids_to_messages.itervalues():
catalog_obj.add(message.id, None, locations=message.locations, auto_comments=message.auto_comments)
return self.write_template(
template_path, catalog_obj, include_obsolete=include_obsolete, include_header=include_header
)
def update(self, template, no_fuzzy_matching=False):
"""Update the catalog based on the given template catalog.
>>> from babel.messages import Catalog
>>> template = Catalog()
>>> template.add('green', locations=[('main.py', 99)])
<Message ...>
>>> template.add('blue', locations=[('main.py', 100)])
<Message ...>
>>> template.add(('salad', 'salads'), locations=[('util.py', 42)])
<Message ...>
>>> catalog = Catalog(locale='de_DE')
>>> catalog.add('blue', u'blau', locations=[('main.py', 98)])
<Message ...>
>>> catalog.add('head', u'Kopf', locations=[('util.py', 33)])
<Message ...>
>>> catalog.add(('salad', 'salads'), (u'Salat', u'Salate'),
... locations=[('util.py', 38)])
<Message ...>
>>> catalog.update(template)
>>> len(catalog)
3
>>> msg1 = catalog['green']
>>> msg1.string
>>> msg1.locations
[('main.py', 99)]
>>> msg2 = catalog['blue']
>>> msg2.string
u'blau'
>>> msg2.locations
[('main.py', 100)]
>>> msg3 = catalog['salad']
>>> msg3.string
(u'Salat', u'Salate')
>>> msg3.locations
[('util.py', 42)]
Messages that are in the catalog but not in the template are removed
from the main collection, but can still be accessed via the `obsolete`
member:
>>> 'head' in catalog
False
>>> catalog.obsolete.values()
[<Message 'head' (flags: [])>]
:param template: the reference catalog, usually read from a POT file
:param no_fuzzy_matching: whether to use fuzzy matching of message IDs
"""
messages = self._messages
remaining = messages.copy()
self._messages = odict()
# Prepare for fuzzy matching
fuzzy_candidates = []
if not no_fuzzy_matching:
fuzzy_candidates = dict([
(self._key_for(msgid), messages[msgid].context)
for msgid in messages if msgid and messages[msgid].string
])
fuzzy_matches = set()
def _merge(message, oldkey, newkey):
message = message.clone()
fuzzy = False
if oldkey != newkey:
fuzzy = True
fuzzy_matches.add(oldkey)
oldmsg = messages.get(oldkey)
if isinstance(oldmsg.id, string_types):
message.previous_id = [oldmsg.id]
else:
message.previous_id = list(oldmsg.id)
else:
oldmsg = remaining.pop(oldkey, None)
message.string = oldmsg.string
if isinstance(message.id, (list, tuple)):
if not isinstance(message.string, (list, tuple)):
fuzzy = True
message.string = tuple(
[message.string] + ([u''] * (len(message.id) - 1))
)
elif len(message.string) != self.num_plurals:
fuzzy = True
message.string = tuple(message.string[:len(oldmsg.string)])
elif isinstance(message.string, (list, tuple)):
fuzzy = True
message.string = message.string[0]
message.flags |= oldmsg.flags
if fuzzy:
message.flags |= set([u'fuzzy'])
self[message.id] = message
for message in template:
if message.id:
key = self._key_for(message.id, message.context)
if key in messages:
_merge(message, key, key)
else:
if no_fuzzy_matching is False:
# do some fuzzy matching with difflib
if isinstance(key, tuple):
matchkey = key[0] # just the msgid, no context
else:
matchkey = key
matches = get_close_matches(matchkey.lower().strip(),
fuzzy_candidates.keys(), 1)
if matches:
newkey = matches[0]
newctxt = fuzzy_candidates[newkey]
if newctxt is not None:
newkey = newkey, newctxt
_merge(message, newkey, key)
continue
self[message.id] = message
for msgid in remaining:
if no_fuzzy_matching or msgid not in fuzzy_matches:
self.obsolete[msgid] = remaining[msgid]
# Make updated catalog's POT-Creation-Date equal to the template
# used to update the catalog
self.creation_date = template.creation_date
def rollback_obsolete(self):
self._messages.update(self.obsolete)
self.obsolete = odict()
def update(self, template, no_fuzzy_matching=False):
"""Update the catalog based on the given template catalog.
>>> from babel.messages import Catalog
>>> template = Catalog()
>>> template.add('green', locations=[('main.py', 99)])
>>> template.add('blue', locations=[('main.py', 100)])
>>> template.add(('salad', 'salads'), locations=[('util.py', 42)])
>>> catalog = Catalog(locale='de_DE')
>>> catalog.add('blue', u'blau', locations=[('main.py', 98)])
>>> catalog.add('head', u'Kopf', locations=[('util.py', 33)])
>>> catalog.add(('salad', 'salads'), (u'Salat', u'Salate'),
... locations=[('util.py', 38)])
>>> catalog.update(template)
>>> len(catalog)
3
>>> msg1 = catalog['green']
>>> msg1.string
>>> msg1.locations
[('main.py', 99)]
>>> msg2 = catalog['blue']
>>> msg2.string
u'blau'
>>> msg2.locations
[('main.py', 100)]
>>> msg3 = catalog['salad']
>>> msg3.string
(u'Salat', u'Salate')
>>> msg3.locations
[('util.py', 42)]
Messages that are in the catalog but not in the template are removed
from the main collection, but can still be accessed via the `obsolete`
member:
>>> 'head' in catalog
False
>>> catalog.obsolete.values()
[<Message 'head' (flags: [])>]
:param template: the reference catalog, usually read from a POT file
:param no_fuzzy_matching: whether to use fuzzy matching of message IDs
"""
messages = self._messages
remaining = messages.copy()
self._messages = odict()
# Prepare for fuzzy matching
fuzzy_candidates = []
if not no_fuzzy_matching:
fuzzy_candidates = [
self._key_for(msgid) for msgid in messages
if msgid and messages[msgid].string
]
fuzzy_matches = set()
def _merge(message, oldkey, newkey):
message = message.clone()
fuzzy = False
if oldkey != newkey:
fuzzy = True
fuzzy_matches.add(oldkey)
oldmsg = messages.get(oldkey)
if isinstance(oldmsg.id, basestring):
message.previous_id = [oldmsg.id]
else:
message.previous_id = list(oldmsg.id)
else:
oldmsg = remaining.pop(oldkey, None)
message.string = oldmsg.string
if isinstance(message.id, (list, tuple)):
if not isinstance(message.string, (list, tuple)):
fuzzy = True
message.string = tuple(
[message.string] + ([u''] * (len(message.id) - 1))
)
elif len(message.string) != self.num_plurals:
fuzzy = True
message.string = tuple(message.string[:len(oldmsg.string)])
elif isinstance(message.string, (list, tuple)):
fuzzy = True
message.string = message.string[0]
message.flags |= oldmsg.flags
if fuzzy:
message.flags |= set([u'fuzzy'])
self[message.id] = message
for message in template:
if message.id:
key = self._key_for(message.id)
if key in messages:
_merge(message, key, key)
else:
if no_fuzzy_matching is False:
# do some fuzzy matching with difflib
matches = get_close_matches(key.lower().strip(),
fuzzy_candidates, 1)
if matches:
_merge(message, matches[0], key)
continue
self[message.id] = message
self.obsolete = odict()
for msgid in remaining:
if no_fuzzy_matching or msgid not in fuzzy_matches:
self.obsolete[msgid] = remaining[msgid]
def _rollback_obsolete(catalog):
catalog._messages.update(catalog.obsolete)
catalog.obsolete = odict()
def update(self, template, no_fuzzy_matching=False):
"""Update the catalog based on the given template catalog.
>>> from babel.messages import Catalog
>>> template = Catalog()
>>> template.add('green', locations=[('main.py', 99)])
<Message ...>
>>> template.add('blue', locations=[('main.py', 100)])
<Message ...>
>>> template.add(('salad', 'salads'), locations=[('util.py', 42)])
<Message ...>
>>> catalog = Catalog(locale='de_DE')
>>> catalog.add('blue', u('blau'), locations=[('main.py', 98)])
<Message ...>
>>> catalog.add('head', u('Kopf'), locations=[('util.py', 33)])
<Message ...>
>>> catalog.add(('salad', 'salads'), (u('Salat'), u('Salate')),
... locations=[('util.py', 38)])
<Message ...>
>>> catalog.update(template)
>>> len(catalog)
3
>>> msg1 = catalog['green']
>>> msg1.string
>>> msg1.locations
[('main.py', 99)]
>>> msg2 = catalog['blue']
>>> print(msg2.string)
blau
>>> msg2.locations
[('main.py', 100)]
>>> msg3 = catalog['salad']
>>> print(msg3.string[0])
Salat
>>> print(msg3.string[1])
Salate
>>> msg3.locations
[('util.py', 42)]
Messages that are in the catalog but not in the template are removed
from the main collection, but can still be accessed via the `obsolete`
member:
>>> 'head' in catalog
False
>>> for v in catalog.obsolete.values():
... print(v)
<Message head (flags: [])>
:param template: the reference catalog, usually read from a POT file
:param no_fuzzy_matching: whether to use fuzzy matching of message IDs
"""
messages = self._messages
remaining = messages.copy()
self._messages = odict()
# Prepare for fuzzy matching
fuzzy_candidates = []
if not no_fuzzy_matching:
fuzzy_candidates = dict([
(self._key_for(msgid), messages[msgid].context)
for msgid in messages if msgid and messages[msgid].string
])
fuzzy_matches = set()
def _merge(message, oldkey, newkey):
message = message.clone()
fuzzy = False
if oldkey != newkey:
fuzzy = True
fuzzy_matches.add(oldkey)
oldmsg = messages.get(oldkey)
if isinstance(oldmsg.id, string_types):
message.previous_id = [oldmsg.id]
else:
message.previous_id = list(oldmsg.id)
else:
oldmsg = remaining.pop(oldkey, None)
message.string = oldmsg.string
if isinstance(message.id, (list, tuple)):
if not isinstance(message.string, (list, tuple)):
fuzzy = True
message.string = tuple([message.string] +
([u('')] * (len(message.id) - 1)))
elif len(message.string) != self.num_plurals:
fuzzy = True
message.string = tuple(message.string[:len(oldmsg.string)])
elif isinstance(message.string, (list, tuple)):
fuzzy = True
message.string = message.string[0]
message.flags |= oldmsg.flags
if fuzzy:
message.flags |= set([u('fuzzy')])
self[message.id] = message
for message in template:
if message.id:
key = self._key_for(message.id, message.context)
if key in messages:
_merge(message, key, key)
else:
if no_fuzzy_matching is False:
# do some fuzzy matching with difflib
if isinstance(key, tuple):
matchkey = key[0] # just the msgid, no context
else:
matchkey = key
matches = get_close_matches(matchkey.lower().strip(),
fuzzy_candidates.keys(), 1)
if matches:
newkey = matches[0]
newctxt = fuzzy_candidates[newkey]
if newctxt is not None:
newkey = newkey, newctxt
_merge(message, newkey, key)
continue
self[message.id] = message
self.obsolete = odict()
for msgid in remaining:
if no_fuzzy_matching or msgid not in fuzzy_matches:
self.obsolete[msgid] = remaining[msgid]
# Make updated catalog's POT-Creation-Date equal to the template
# used to update the catalog
self.creation_date = template.creation_date
def extract(self, include_obsolete=False, localized=False, paths=None,
include_header=False, locales=None, use_fuzzy_matching=False):
env = self.pod.get_jinja_env()
all_locales = set(list(self.pod.list_locales()))
message_ids_to_messages = {}
paths_to_messages = collections.defaultdict(set)
paths_to_locales = collections.defaultdict(set)
comment_tags = [
':',
]
options = {
'extensions': ','.join(env.extensions.keys()),
'silent': 'false',
}
# Extract from content files.
def callback(doc, item, key, unused_node):
# Verify that the fields we're extracting are fields for a document
# that's in the default locale. If not, skip the document.
_handle_field(doc.pod_path, item, key, unused_node)
def _add_existing_message(msgid, locations, auto_comments=None,
context=None, path=None):
existing_message = message_ids_to_messages.get(msgid)
auto_comments = [] if auto_comments is None else auto_comments
if existing_message:
message_ids_to_messages[msgid].locations.extend(locations)
paths_to_messages[path].add(existing_message)
else:
message = catalog.Message(
msgid,
None,
auto_comments=auto_comments,
context=context,
locations=locations)
paths_to_messages[path].add(message)
message_ids_to_messages[message.id] = message
def _handle_field(path, item, key, node):
if (not key
or not isinstance(item, basestring)
or not isinstance(key, basestring)
or not key.endswith('@')):
return
# Support gettext "extracted comments" on tagged fields. This is
# consistent with extracted comments in templates, which follow
# the format "{#: Extracted comment. #}". An example:
# field@: Message.
# field@#: Extracted comment for field@.
auto_comments = []
if isinstance(node, dict):
auto_comment = node.get('{}#'.format(key))
if auto_comment:
auto_comments.append(auto_comment)
locations = [(path, 0)]
_add_existing_message(
msgid=item,
auto_comments=auto_comments,
locations=locations,
path=path)
for collection in self.pod.list_collections():
text = 'Extracting collection: {}'.format(collection.pod_path)
self.pod.logger.info(text)
# Extract from blueprint.
utils.walk(collection.tagged_fields, lambda *args: callback(collection, *args))
# Extract from docs in collection.
for doc in collection.docs(include_hidden=True):
if not self._should_extract_as_babel(paths, doc.pod_path):
continue
tagged_fields = doc.get_tagged_fields()
utils.walk(tagged_fields, lambda *args: callback(doc, *args))
paths_to_locales[doc.pod_path].update(doc.locales)
all_locales.update(doc.locales)
# Extract from podspec.
config = self.pod.get_podspec().get_config()
podspec_path = '/podspec.yaml'
if self._should_extract_as_babel(paths, podspec_path):
self.pod.logger.info('Extracting podspec: {}'.format(podspec_path))
utils.walk(config, lambda *args: _handle_field(podspec_path, *args))
# Extract from content and views.
pod_files = [os.path.join('/views', path)
for path in self.pod.list_dir('/views/')]
pod_files += [os.path.join('/content', path)
for path in self.pod.list_dir('/content/')]
pod_files += [os.path.join('/data', path)
for path in self.pod.list_dir('/data/')]
for pod_path in pod_files:
if self._should_extract_as_csv(paths, pod_path):
rows = utils.get_rows_from_csv(self.pod, pod_path)
self.pod.logger.info('Extracting: {}'.format(pod_path))
for row in rows:
for i, parts in enumerate(row.iteritems()):
key, val = parts
if key.endswith('@'):
locations = [(pod_path, i)]
_add_existing_message(
msgid=val,
locations=locations,
path=pod_path)
elif self._should_extract_as_babel(paths, pod_path):
if pod_path.startswith('/data') and pod_path.endswith(('.yaml', '.yml')):
self.pod.logger.info('Extracting: {}'.format(pod_path))
content = self.pod.read_file(pod_path)
fields = utils.load_yaml(content, pod=self.pod)
utils.walk(fields, lambda *args: _handle_field(pod_path, *args))
continue
pod_locales = paths_to_locales.get(pod_path)
if pod_locales:
text = 'Extracting: {} ({} locales)'
text = text.format(pod_path, len(pod_locales))
self.pod.logger.info(text)
else:
self.pod.logger.info('Extracting: {}'.format(pod_path))
fp = self.pod.open_file(pod_path)
try:
all_parts = extract.extract(
'jinja2.ext.babel_extract', fp, options=options,
comment_tags=comment_tags)
for parts in all_parts:
lineno, string, comments, context = parts
locations = [(pod_path, lineno)]
_add_existing_message(
msgid=string,
auto_comments=comments,
context=context,
locations=locations,
path=pod_path)
except tokenize.TokenError:
self.pod.logger.error('Problem extracting: {}'.format(pod_path))
raise
# Localized message catalogs.
if localized:
for locale in all_locales:
if locales and locale not in locales:
continue
localized_catalog = self.get(locale)
if not include_obsolete:
localized_catalog.obsolete = babel_util.odict()
for message in list(localized_catalog):
if message.id not in message_ids_to_messages:
localized_catalog.delete(message.id, context=message.context)
catalog_to_merge = catalog.Catalog()
for path, message_items in paths_to_messages.iteritems():
locales_with_this_path = paths_to_locales.get(path)
if locales_with_this_path and locale not in locales_with_this_path:
continue
for message in message_items:
translation = None
existing_message = localized_catalog.get(message.id)
if existing_message:
translation = existing_message.string
catalog_to_merge.add(
message.id, translation, locations=message.locations,
auto_comments=message.auto_comments, flags=message.flags,
user_comments=message.user_comments, context=message.context,
lineno=message.lineno, previous_id=message.previous_id)
localized_catalog.update_using_catalog(
catalog_to_merge, use_fuzzy_matching=use_fuzzy_matching)
localized_catalog.save(include_header=include_header)
missing = localized_catalog.list_untranslated()
num_messages = len(localized_catalog)
num_translated = num_messages - len(missing)
text = 'Saved: /{path} ({num_translated}/{num_messages})'
self.pod.logger.info(
text.format(path=localized_catalog.pod_path,
num_translated=num_translated,
num_messages=num_messages))
return
# Global (or missing, specified by -o) message catalog.
template_path = self.template_path
catalog_obj, _ = self._get_or_create_catalog(template_path)
if not include_obsolete:
catalog_obj.obsolete = babel_util.odict()
for message in list(catalog_obj):
catalog_obj.delete(message.id, context=message.context)
for message in message_ids_to_messages.itervalues():
if message.id:
catalog_obj.add(message.id, None, locations=message.locations,
auto_comments=message.auto_comments)
return self.write_template(
template_path, catalog_obj, include_obsolete=include_obsolete,
include_header=include_header)
