def _handle_pairs_of_parts_and_bodies(self):
try:
locales_to_bodies = {}
locales_to_fields = {}
locale = self.doc._locale_kwarg
default_locale = None
split_content = Format.split_front_matter(self.content)
for part, body in utils.every_two(split_content):
fields = utils.load_yaml(part, pod=self.doc.pod)
if '$locale' in fields and '$locales' in fields:
text = 'You must specify either $locale or $locales, not both.'
raise BadFormatError(text)
if '$locales' in fields:
doc_locales = fields['$locales']
else:
doc_locales = [fields.get('$locale', default_locale)]
for doc_locale in doc_locales:
locales_to_fields[doc_locale] = fields
locales_to_bodies[doc_locale] = body
fields = locales_to_fields.get(default_locale)
if locale in locales_to_fields:
localized_fields = locales_to_fields[locale]
if fields is None:
fields = {}
fields.update(localized_fields)
default_body = locales_to_bodies.get(default_locale)
self.body = locales_to_bodies.get(locale, default_body)
self.body = self.body.strip() if self.body is not None else None
self.fields = fields
except (yaml.composer.ComposerError, yaml.scanner.ScannerError) as e:
message = 'Error parsing {}: {}'.format(self.doc.pod_path, e)
logging.exception(message)
raise BadFormatError(message)
def _load_yaml(self, part):
try:
return utils.load_yaml(part, doc=self.doc, pod=self.doc.pod)
except (yaml.parser.ParserError, yaml.composer.ComposerError,
yaml.scanner.ScannerError) as e:
message = 'Error parsing {}: {}'.format(self.doc.pod_path, e)
raise BadFormatError(message)
def _load_yaml(self, part):
try:
return utils.load_yaml(part, doc=self.doc, pod=self.doc.pod)
except (yaml.parser.ParserError,
yaml.composer.ComposerError,
yaml.scanner.ScannerError) as e:
message = 'Error parsing {}: {}'.format(self.doc.pod_path, e)
raise BadFormatError(message)
def load(self):
try:
if not self.has_front_matter:
self.fields = utils.load_yaml(self.content, pod=self.doc.pod)
self.body = self.content
return
self._handle_pairs_of_parts_and_bodies()
except (yaml.composer.ComposerError, yaml.scanner.ScannerError) as e:
message = 'Error parsing {}: {}'.format(self.doc.pod_path, e)
logging.exception(message)
raise BadFormatError(message)
def load(self):
try:
if self._has_front_matter:
self._handle_pairs_of_parts_and_bodies()
else:
self.body = self.content
self.fields = utils.load_yaml(
self.content, doc=self.doc, pod=self.doc.pod) or {}
except (yaml.composer.ComposerError, yaml.scanner.ScannerError) as e:
message = 'Error parsing {}: {}'.format(self.doc.pod_path, e)
logging.exception(message)
raise BadFormatError(message)
def load(self):
if not self.has_front_matter:
self.fields = utils.load_yaml(self.content, pod=self.doc.pod)
self.body = self.content
return
locales_to_fields = {}
locales_to_bodies = {}
locale = str(self.doc.locale)
default_locale = str(self.doc.default_locale)
for part in Format.split_front_matter(self.content):
fields = utils.load_yaml(part, pod=self.doc.pod)
doc_locale = fields.get('$locale', default_locale)
locales_to_fields[doc_locale] = fields
locales_to_bodies[doc_locale] = part
fields = locales_to_fields.get(default_locale)
if locale in locales_to_fields:
localized_fields = locales_to_fields[locale]
if fields is None:
fields = {}
fields.update(localized_fields)
self.body = locales_to_bodies.get(locale, locales_to_bodies.get(default_locale))
self.body = self.body.strip() if self.body is not None else None
self.fields = fields
def _load_yaml(self, raw_yaml):
try:
return utils.load_yaml(
raw_yaml, doc=self._doc, pod=self._doc.pod,
untag_params={
'env': untag.UntagParamRegex(self._doc.pod.env.name),
'locale': untag.UntagParamLocaleRegex.from_pod(
self._doc.pod, self._doc.collection),
})
except (yaml.composer.ComposerError,
yaml.parser.ParserError,
yaml.reader.ReaderError,
yaml.scanner.ScannerError) as error:
message = 'Error parsing {}: {}'.format(self._doc.pod_path, error)
raise BadFormatError(message)
def _handle_pairs_of_parts_and_bodies(self):
locales_to_bodies = {}
locales_to_fields = {}
locale = str(self.doc.locale)
default_locale = str(self.doc.default_locale)
for part, body in utils.every_two(Format.split_front_matter(self.content)):
fields = utils.load_yaml(part, pod=self.doc.pod)
doc_locale = fields.get('$locale', default_locale)
locales_to_fields[doc_locale] = fields
locales_to_bodies[doc_locale] = body
fields = locales_to_fields.get(default_locale)
if locale in locales_to_fields:
localized_fields = locales_to_fields[locale]
if fields is None:
fields = {}
fields.update(localized_fields)
self.body = locales_to_bodies.get(locale, locales_to_bodies.get(default_locale))
self.body = self.body.strip() if self.body is not None else None
self.fields = fields
def raw_data(self):
if not self._raw_front_matter:
return {}
return utils.load_yaml(
self._raw_front_matter, doc=self._doc, pod=self._doc.pod)
def extract(self, include_obsolete=False, localized=False, paths=None,
include_header=False, locales=None, use_fuzzy_matching=False):
env = self.pod.get_jinja_env()
all_locales = set(list(self.pod.list_locales()))
message_ids_to_messages = {}
paths_to_messages = collections.defaultdict(set)
paths_to_locales = collections.defaultdict(set)
comment_tags = [
':',
]
options = {
'extensions': ','.join(env.extensions.keys()),
'silent': 'false',
}
# Extract from content files.
def callback(doc, item, key, unused_node):
# Verify that the fields we're extracting are fields for a document
# that's in the default locale. If not, skip the document.
_handle_field(doc.pod_path, item, key, unused_node)
def _add_existing_message(msgid, locations, auto_comments=None,
context=None, path=None):
existing_message = message_ids_to_messages.get(msgid)
auto_comments = [] if auto_comments is None else auto_comments
if existing_message:
message_ids_to_messages[msgid].locations.extend(locations)
paths_to_messages[path].add(existing_message)
else:
message = catalog.Message(
msgid,
None,
auto_comments=auto_comments,
context=context,
locations=locations)
paths_to_messages[path].add(message)
message_ids_to_messages[message.id] = message
def _handle_field(path, item, key, node):
if (not key
or not isinstance(item, basestring)
or not isinstance(key, basestring)
or not key.endswith('@')):
return
# Support gettext "extracted comments" on tagged fields. This is
# consistent with extracted comments in templates, which follow
# the format "{#: Extracted comment. #}". An example:
# field@: Message.
# field@#: Extracted comment for field@.
auto_comments = []
if isinstance(node, dict):
auto_comment = node.get('{}#'.format(key))
if auto_comment:
auto_comments.append(auto_comment)
locations = [(path, 0)]
_add_existing_message(
msgid=item,
auto_comments=auto_comments,
locations=locations,
path=path)
for collection in self.pod.list_collections():
text = 'Extracting collection: {}'.format(collection.pod_path)
self.pod.logger.info(text)
# Extract from blueprint.
utils.walk(collection.tagged_fields, lambda *args: callback(collection, *args))
# Extract from docs in collection.
for doc in collection.docs(include_hidden=True):
if not self._should_extract_as_babel(paths, doc.pod_path):
continue
tagged_fields = doc.get_tagged_fields()
utils.walk(tagged_fields, lambda *args: callback(doc, *args))
paths_to_locales[doc.pod_path].update(doc.locales)
all_locales.update(doc.locales)
# Extract from podspec.
config = self.pod.get_podspec().get_config()
podspec_path = '/podspec.yaml'
if self._should_extract_as_babel(paths, podspec_path):
self.pod.logger.info('Extracting podspec: {}'.format(podspec_path))
utils.walk(config, lambda *args: _handle_field(podspec_path, *args))
# Extract from content and views.
pod_files = [os.path.join('/views', path)
for path in self.pod.list_dir('/views/')]
pod_files += [os.path.join('/content', path)
for path in self.pod.list_dir('/content/')]
pod_files += [os.path.join('/data', path)
for path in self.pod.list_dir('/data/')]
for pod_path in pod_files:
if self._should_extract_as_csv(paths, pod_path):
rows = utils.get_rows_from_csv(self.pod, pod_path)
self.pod.logger.info('Extracting: {}'.format(pod_path))
for row in rows:
for i, parts in enumerate(row.iteritems()):
key, val = parts
if key.endswith('@'):
locations = [(pod_path, i)]
_add_existing_message(
msgid=val,
locations=locations,
path=pod_path)
elif self._should_extract_as_babel(paths, pod_path):
if pod_path.startswith('/data') and pod_path.endswith(('.yaml', '.yml')):
self.pod.logger.info('Extracting: {}'.format(pod_path))
content = self.pod.read_file(pod_path)
fields = utils.load_yaml(content, pod=self.pod)
utils.walk(fields, lambda *args: _handle_field(pod_path, *args))
continue
pod_locales = paths_to_locales.get(pod_path)
if pod_locales:
text = 'Extracting: {} ({} locales)'
text = text.format(pod_path, len(pod_locales))
self.pod.logger.info(text)
else:
self.pod.logger.info('Extracting: {}'.format(pod_path))
fp = self.pod.open_file(pod_path)
try:
all_parts = extract.extract(
'jinja2.ext.babel_extract', fp, options=options,
comment_tags=comment_tags)
for parts in all_parts:
lineno, string, comments, context = parts
locations = [(pod_path, lineno)]
_add_existing_message(
msgid=string,
auto_comments=comments,
context=context,
locations=locations,
path=pod_path)
except tokenize.TokenError:
self.pod.logger.error('Problem extracting: {}'.format(pod_path))
raise
# Localized message catalogs.
if localized:
for locale in all_locales:
if locales and locale not in locales:
continue
localized_catalog = self.get(locale)
if not include_obsolete:
localized_catalog.obsolete = babel_util.odict()
for message in list(localized_catalog):
if message.id not in message_ids_to_messages:
localized_catalog.delete(message.id, context=message.context)
catalog_to_merge = catalog.Catalog()
for path, message_items in paths_to_messages.iteritems():
locales_with_this_path = paths_to_locales.get(path)
if locales_with_this_path and locale not in locales_with_this_path:
continue
for message in message_items:
translation = None
existing_message = localized_catalog.get(message.id)
if existing_message:
translation = existing_message.string
catalog_to_merge.add(
message.id, translation, locations=message.locations,
auto_comments=message.auto_comments, flags=message.flags,
user_comments=message.user_comments, context=message.context,
lineno=message.lineno, previous_id=message.previous_id)
localized_catalog.update_using_catalog(
catalog_to_merge, use_fuzzy_matching=use_fuzzy_matching)
localized_catalog.save(include_header=include_header)
missing = localized_catalog.list_untranslated()
num_messages = len(localized_catalog)
num_translated = num_messages - len(missing)
text = 'Saved: /{path} ({num_translated}/{num_messages})'
self.pod.logger.info(
text.format(path=localized_catalog.pod_path,
num_translated=num_translated,
num_messages=num_messages))
return
# Global (or missing, specified by -o) message catalog.
template_path = self.template_path
catalog_obj, _ = self._get_or_create_catalog(template_path)
if not include_obsolete:
catalog_obj.obsolete = babel_util.odict()
for message in list(catalog_obj):
catalog_obj.delete(message.id, context=message.context)
for message in message_ids_to_messages.itervalues():
if message.id:
catalog_obj.add(message.id, None, locations=message.locations,
auto_comments=message.auto_comments)
return self.write_template(
template_path, catalog_obj, include_obsolete=include_obsolete,
include_header=include_header)