def run(self):
mappings = self._get_mappings()
with open(self.output_file, 'wb') as outfile:
catalog = Catalog(project=self.project,
version=self.version,
msgid_bugs_address=self.msgid_bugs_address,
copyright_holder=self.copyright_holder,
charset=self.charset)
for path, method_map, options_map in mappings:
def callback(filename, method, options):
if method == 'ignore':
return
# If we explicitly provide a full filepath, just use that.
# Otherwise, path will be the directory path and filename
# is the relative path from that dir to the file.
# So we can join those to get the full filepath.
if os.path.isfile(path):
filepath = path
else:
filepath = os.path.normpath(os.path.join(path, filename))
optstr = ''
if options:
optstr = ' (%s)' % ', '.join(['%s="%s"' % (k, v) for
k, v in options.items()])
self.log.info('extracting messages from %s%s', filepath, optstr)
if os.path.isfile(path):
current_dir = os.getcwd()
extracted = check_and_call_extract_file(
path, method_map, options_map,
callback, self.keywords, self.add_comments,
self.strip_comments, current_dir
)
else:
extracted = extract_from_dir(
path, method_map, options_map,
keywords=self.keywords,
comment_tags=self.add_comments,
callback=callback,
strip_comment_tags=self.strip_comments
)
for filename, lineno, message, comments, context in extracted:
if os.path.isfile(path):
filepath = filename # already normalized
else:
filepath = os.path.normpath(os.path.join(path, filename))
catalog.add(message, None, [(filepath, lineno)],
auto_comments=comments, context=context)
self.log.info('writing PO template file to %s', self.output_file)
write_po(outfile, catalog, width=self.width,
no_location=self.no_location,
omit_header=self.omit_header,
sort_output=self.sort_output,
sort_by_file=self.sort_by_file,
include_lineno=self.include_lineno)
def extract(self, src_path='.', charset='utf-8', locale=None, **kwargs):
"""Extracts translatable messages from sources. This function is based
on the extract function of the `pybabel` command, which is not part of
the public API of `babel`. Only the public API of `babel` is used here.
:param src_path: base path of the source tree, default is the current
path.
:param charset: see the `babel.messages.catalog.Catalog` docs. Default
is `utf-8`.
:param locale: see the `babel.messages.catalog.Catalog` docs. Default
is `None`.
Other optional keyword parameters are passed to
`babel.messages.extract.extract_from_dir` see `babel` public API docs.
"""
#: This is the babel.messages.catalog.Catalog to contain the
#: extracted messages
self.catalog = Catalog(charset=charset, locale=locale)
if not pth.isdir(src_path):
raise IOError('{} is not a directory'.format(src_path))
#: Extracts the data from source in a low level format. This is
#: the only way present in babel's public API.
extracted = extract_from_dir(src_path, **kwargs)
#: Constructs the catalog from the raw extracted data.
#: Based on the source code of pybabel:
#: babel.messages.frontend.extract_messages.run
for filename, lineno, message, comments, context in extracted:
self.catalog.add(message, None, [(filename, lineno)],
auto_comments=comments, context=context)
def extract(self, src_path='.', charset='utf-8', locale=None, **kwargs):
"""Extracts translatable messages from sources. This function is based
on the extract function of the `pybabel` command, which is not part of
the public API of `babel`. Only the public API of `babel` is used here.
:param src_path: base path of the source tree, default is the current
path.
:param charset: see the `babel.messages.catalog.Catalog` docs. Default
is `utf-8`.
:param locale: see the `babel.messages.catalog.Catalog` docs. Default
is `None`.
Other optional keyword parameters are passed to
`babel.messages.extract.extract_from_dir` see `babel` public API docs.
"""
#: This is the babel.messages.catalog.Catalog to contain the
#: extracted messages
self.catalog = Catalog(charset=charset, locale=locale)
if not pth.isdir(src_path):
raise IOError('{} is not a directory'.format(src_path))
#: Extracts the data from source in a low level format. This is
#: the only way present in babel's public API.
extracted = extract_from_dir(src_path, **kwargs)
#: Constructs the catalog from the raw extracted data.
#: Based on the source code of pybabel:
#: babel.messages.frontend.extract_messages.run
for filename, lineno, message, comments, context in extracted:
self.catalog.add(message, None, [(filename, lineno)],
auto_comments=comments, context=context)
def run(self):
mappings = self._get_mappings()
with open(self.output_file, 'wb') as outfile:
catalog = Catalog(project=self.project,
version=self.version,
msgid_bugs_address=self.msgid_bugs_address,
copyright_holder=self.copyright_holder,
charset=self.charset)
for path, method_map, options_map in mappings:
def callback(filename, method, options):
if method == 'ignore':
return
# If we explicitly provide a full filepath, just use that.
# Otherwise, path will be the directory path and filename
# is the relative path from that dir to the file.
# So we can join those to get the full filepath.
if os.path.isfile(path):
filepath = path
else:
filepath = os.path.normpath(os.path.join(path, filename))
optstr = ''
if options:
optstr = ' (%s)' % ', '.join(['%s="%s"' % (k, v) for
k, v in options.items()])
self.log.info('extracting messages from %s%s', filepath, optstr)
if os.path.isfile(path):
current_dir = os.getcwd()
extracted = check_and_call_extract_file(
path, method_map, options_map,
callback, self.keywords, self.add_comments,
self.strip_comments, current_dir
)
else:
extracted = extract_from_dir(
path, method_map, options_map,
keywords=self.keywords,
comment_tags=self.add_comments,
callback=callback,
strip_comment_tags=self.strip_comments
)
for fname, lineno, msg, comments, context, flags in extracted:
if os.path.isfile(path):
filepath = fname # already normalized
else:
filepath = os.path.normpath(os.path.join(path, fname))
catalog.add(msg, None, [(filepath, lineno)], flags=flags,
auto_comments=comments, context=context)
self.log.info('writing PO template file to %s', self.output_file)
write_po(outfile, catalog, width=self.width,
no_location=self.no_location,
omit_header=self.omit_header,
sort_output=self.sort_output,
sort_by_file=self.sort_by_file,
include_lineno=self.include_lineno)
def extract(self):
if not self.input_dirs:
return
logger.info('Processing catalog "%s"', self.name)
catalog = Catalog()
for dirpath in self.input_dirs:
extracted = extract_from_dir(
str(dirpath),
self.method_map,
self.options_map,
)
for filename, lineno, message, comments, context in extracted:
fpath = dirpath.join(filename)
if self.root_dir:
fpath = os.path.relpath(fpath, self.root_dir)
logger.info('Extracting messages from %s', fpath)
catalog.add(
message,
None,
[(fpath, lineno)],
auto_comments=comments,
context=context,
)
logger.info('Writing PO template file to %s', str(self.pot_file))
self.pot_file.makedirs()
with self.pot_file.open('w') as pot_fp:
write_po(pot_fp, catalog, omit_header=True)
def extract_to_catalog(catalog, data_iterator, keywords, comment_tags,
strip_comment_tags, log=log):
for dirname, method_map, options_map in data_iterator:
if not os.path.isdir(dirname):
raise ConfigureError('%r is not a directory' % dirname)
def callback(filename, method, options):
if method == 'ignore':
return
filepath = _gen_filepath(dirname, filename)
optstr = _gen_optstr(options)
log.info('extracting messages from %s%s', filepath, optstr)
extracted = extract_from_dir(dirname, method_map, options_map,
keywords=keywords,
comment_tags=comment_tags,
callback=callback,
strip_comment_tags=strip_comment_tags)
# Add extracted strings to catalog
for filename, lineno, message, comments, context in extracted:
filepath = _gen_filepath(dirname, filename)
catalog.add(message, None, [(filepath, lineno)],
auto_comments=comments, context=context)
def extract_messages(dirs):
catalog = Catalog(project='Open Library',
copyright_holder='Internet Archive')
METHODS = [("**.py", "python"),
("**.html", "openlibrary.i18n:extract_templetor")]
COMMENT_TAGS = ["NOTE:"]
for d in dirs:
if '.html' in d:
extracted = [(d, ) + extract for extract in extract_from_file(
"openlibrary.i18n:extract_templetor", d)]
else:
extracted = extract_from_dir(d,
METHODS,
comment_tags=COMMENT_TAGS,
strip_comment_tags=True)
for filename, lineno, message, comments, context in extracted:
catalog.add(message,
None, [(filename, lineno)],
auto_comments=comments)
path = os.path.join(root, 'messages.pot')
f = open(path, 'w')
write_po(f, catalog)
f.close()
print('wrote template to', path)
def extract_messages(dirs: list[str]):
catalog = Catalog(project='Open Library',
copyright_holder='Internet Archive')
METHODS = [("**.py", "python"),
("**.html", "openlibrary.i18n:extract_templetor")]
COMMENT_TAGS = ["NOTE:"]
for d in dirs:
extracted = extract_from_dir(d,
METHODS,
comment_tags=COMMENT_TAGS,
strip_comment_tags=True)
counts: dict[str, int] = {}
for filename, lineno, message, comments, context in extracted:
counts[filename] = counts.get(filename, 0) + 1
catalog.add(message,
None, [(filename, lineno)],
auto_comments=comments)
for filename, count in counts.items():
path = filename if d == filename else os.path.join(d, filename)
print(f"{count}\t{path}", file=sys.stderr)
path = os.path.join(root, 'messages.pot')
f = open(path, 'wb')
write_po(f, catalog)
f.close()
print('wrote template to', path)
def extract_messages(dirs):
catalog = Catalog(
project='Open Library',
copyright_holder='Internet Archive'
)
METHODS = [
("**.py", "python"),
("**.html", "openlibrary.i18n:extract_templetor")
]
COMMENT_TAGS = ["NOTE:"]
for d in dirs:
if '.html' in d:
extracted = [(d,) + extract for extract in extract_from_file("openlibrary.i18n:extract_templetor", d)]
else:
extracted = extract_from_dir(d, METHODS, comment_tags=COMMENT_TAGS, strip_comment_tags=True)
for filename, lineno, message, comments, context in extracted:
catalog.add(message, None, [(filename, lineno)], auto_comments=comments)
path = os.path.join(root, 'messages.pot')
f = open(path, 'w')
write_po(f, catalog)
f.close()
print('wrote template to', path)
def extract_and_update_locale_dir(dirname, mapping):
locale_dir = os.path.join(dirname, 'locale')
for domain in ('django', 'djangojs'):
extracted_catalog = Catalog()
extracted = extract_from_dir(
dirname=dirname,
method_map=mapping[domain]['method_map'],
options_map=mapping[domain]['options_map']
)
for filename, lineno, message, comments, context in extracted:
extracted_catalog.add(message, None, [(filename, lineno)], auto_comments=comments, context=context)
for locale, language in settings.LANGUAGES:
po_path = os.path.join(locale_dir, locale, 'LC_MESSAGES', '%s.po' % domain)
if os.path.exists(po_path):
with open(po_path, 'rb') as po_fileobj:
catalog = read_po(po_fileobj, locale=locale, domain=domain)
if catalog._messages != extracted_catalog._messages:
catalog.update(extracted_catalog, no_fuzzy_matching=True, update_header_comment=False)
with open(po_path, 'wb') as po_fileobj:
write_po(po_fileobj, catalog, ignore_obsolete=True)
elif extracted_catalog._messages:
extracted_catalog.locale = Locale(locale)
extracted_catalog.domain = domain
extracted_catalog.project = 'Mesto.UA'
extracted_catalog.copyright_holder = 'Mesto.UA'
extracted_catalog.fuzzy = False
touch_directory(po_path)
with open(po_path, 'wb') as po_fileobj:
write_po(po_fileobj, extracted_catalog)
def get_message_catalog():
cat = catalog.Catalog(fuzzy = False, charset = 'utf-8')
# we need to retrieve all translatable strings within the source
print("Parsing source for translatable strings...")
tokens = extract.extract_from_dir(
dirname = os.path.join(get_script_directory(), 'stereopoly'),
)
for token in tokens:
cat.add(
token[2],
#locations = (token[0], token[1], ), # currently bricks with 2.6.0
user_comments = token[3],
context = token[4]
)
print("Getting translatable strings from database...")
session = db.setup()()
boards = session.query(db.Board).all()
for b in boards:
for t in b.get_translatables():
cat.add(
t['id'],
user_comments = t['user_comments']
)
session.close()
return cat
def extract(self, force=False):
"""
Extract translation strings from sources directory with extract rules then
create the template catalog with finded translation strings
Only proceed if the template catalog does not exists yet or if
``force`` argument is ``True`` (this will overwrite previous existing
POT file)
TODO: actually from the CLI usage this only update POT file when he does not
exist, else it keeps untouched, even if there changes or adds in translations
"""
if force or not self.check_template_path():
self.logger.info('Proceeding to extraction to update the template catalog (POT)')
self._catalog_template = Catalog(project=self.settings.SITE_NAME, header_comment=self.header_comment)
# Follow all paths to search for pattern to extract
for extract_path in self.settings.I18N_EXTRACT_SOURCES:
self.logger.debug('Searching for pattern to extract in : {0}'.format(extract_path))
extracted = extract_from_dir(dirname=extract_path, method_map=self.settings.I18N_EXTRACT_MAP, options_map=self.settings.I18N_EXTRACT_OPTIONS)
# Proceed to extract from given path
for filename, lineno, message, comments, context in extracted:
filepath = os.path.normpath(os.path.join(os.path.basename(self.settings.SOURCES_DIR), filename))
self._catalog_template.add(message, None, [(filepath, lineno)], auto_comments=comments, context=context)
outfile = open(self.get_template_path(), 'wb')
write_po(outfile, self._catalog_template)
outfile.close()
return self._catalog_template
def cmd_extract(args):
for cident, _ in components_and_locales(args):
module = import_module(pkginfo.comp_mod(cident))
modpath = module.__path__[0]
catalog = Catalog()
method_map, options_map = get_mappings()
def log_callback(filename, method, options):
if method != 'ignore':
filepath = os.path.normpath(os.path.join(modpath, filename))
logger.debug('Extracting messages from %s', filepath)
extracted = extract_from_dir(modpath,
method_map,
options_map,
callback=log_callback)
for filename, lineno, message, comments, context in extracted:
catalog.add(message,
None, [(filename, lineno)],
auto_comments=comments,
context=context)
logger.info("%d messages extracted from component [%s]", len(catalog),
cident)
outfn = catalog_filename(cident, None, ext='pot', mkdir=True)
logger.debug("Writing POT-file to %s", outfn)
with io.open(str(outfn), 'wb') as outfd:
write_po(outfd, catalog, ignore_obsolete=True, omit_header=True)
def babel_extract_messages(*dirnames: str,
project: str = app.cfg.name,
domain: str = self.cfg.domain,
locations: bool = True,
charset: str = 'utf-8',
locale: str = self.cfg.default_locale):
"""Extract messages from source code.
:param charset: charset to use in the output
:param domain: set domain name for locales
:param project: set project name in output
:param version: set project version in output
:param locations: add message locations
"""
dirs = [d for d in dirnames if os.path.isdir(d)]
catalog = Catalog(locale=locale, project=project, charset=charset)
for dname in dirs:
for filename, lineno, message, comments, context in extract_from_dir(
dname,
method_map=self.cfg.sources_map,
options_map=self.cfg.options_map):
lines = []
if locations:
filepath = os.path.normpath(
os.path.join(dname, filename))
lines = [(filepath, lineno)]
catalog.add(message,
None,
lines,
auto_comments=comments,
context=context)
locales_dir = self.cfg.locale_folders[0]
output = os.path.join(locales_dir, locale, 'LC_MESSAGES',
'%s.po' % domain)
if os.path.exists(output):
with open(output, 'rb') as f:
template = read_po(f, locale=locale, charset=charset)
template.update(catalog)
catalog = template
if not os.path.exists(os.path.dirname(output)):
os.makedirs(os.path.dirname(output))
logger.info('writing PO template file to %s', output)
outfile = open(output, 'wb')
try:
write_po(outfile,
catalog,
include_previous=True,
sort_output=not locations,
sort_by_file=locations)
finally:
outfile.close()
def _get_translatable_strings(self) -> dict:
strings_list = extract.extract_from_dir(dirname=str(self.sources_dir))
strings_dict = {}
for filename, line_num, msg, _, context in strings_list:
if msg in strings_dict:
strings_dict[msg].append((filename, line_num))
else:
strings_dict[msg] = [(filename, line_num)]
return strings_dict
def set_nereid(self):
"""
There are messages within the tryton code used in flash messages,
returned responses etc. This is spread over the codebase and this
function extracts the translation strings from code of installed
modules.
"""
pool = Pool()
Translation = pool.get('ir.translation')
to_create = []
for module, directory in self._get_installed_module_directories():
# skip messages from test files
if 'tests' in directory:
continue
for (filename, lineno, messages, comments, context) in \
extract_from_dir(directory,):
if isinstance(messages, basestring):
# messages could be a tuple if the function is ngettext
# where the messages for singular and plural are given as
# a tuple.
#
# So convert basestrings to tuples
messages = (messages, )
for message in messages:
translations = Translation.search([
('lang', '=', 'en_US'),
('type', '=', 'nereid'),
('name', '=', filename),
('src', '=', message),
('module', '=', module),
],
limit=1)
if translations:
continue
to_create.append({
'name':
filename,
'res_id':
lineno,
'lang':
'en_US',
'src':
message,
'type':
'nereid',
'module':
module,
'comments':
comments and '\n'.join(comments) or None,
})
if to_create:
Translation.create(to_create)
def set_wtforms(self):
"""
There are some messages in WTForms which are provided by the framework,
namely default validator messages and errors occuring during the
processing (data coercion) stage. For example, in the case of the
IntegerField, if someone entered a value which was not valid as
an integer, then a message like “Not a valid integer value” would be
displayed.
"""
pool = Pool()
Translation = pool.get('ir.translation')
to_create = []
for (filename, lineno, messages, comments, context) in \
extract_from_dir(os.path.dirname(wtforms.__file__)):
if isinstance(messages, basestring):
# messages could be a tuple if the function is ngettext
# where the messages for singular and plural are given as
# a tuple.
#
# So convert basestrings to tuples
messages = (messages, )
for message in messages:
translations = Translation.search([
('lang', '=', 'en_US'),
('type', '=', 'wtforms'),
('name', '=', filename),
('src', '=', message),
('module', '=', 'nereid'),
],
limit=1)
if translations:
continue
to_create.append({
'name':
filename,
'res_id':
lineno,
'lang':
'en_US',
'src':
message,
'type':
'wtforms',
'module':
'nereid',
'comments':
comments and '\n'.join(comments) or None,
})
if to_create:
Translation.create(to_create)
def run_gettext(dirname, for_js):
catalog = Catalog()
for filename, lineno, message, comments, context in extract_from_dir(
dirname,
method_map=[('**.js', 'javascript')] if for_js else [('**.py', 'python')]
):
catalog.add(message, None, [(filename, lineno)],
auto_comments=comments, context=context)
sio = cStringIO.StringIO()
write_po(sio, catalog)
return sio.getvalue()
def cmd_extract(args):
pkginfo = load_pkginfo(args)
for cident, cdefn in pkginfo['components'].items():
if args.component is not None and cident not in args.component:
continue
if isinstance(cdefn, six.string_types):
cmod = cdefn
else:
cmod = cdefn['module']
module = import_module(cmod)
modpath = module.__path__[0]
dist = get_distribution(args.package)
meta = dict(message_from_string(dist.get_metadata('PKG-INFO')))
catalog = Catalog(project=args.package,
version=dist.version,
copyright_holder=meta.get('Author'),
msgid_bugs_address=meta.get('Author-email'),
fuzzy=False,
charset='utf-8')
method_map, options_map = get_mappings()
def log_callback(filename, method, options):
if method != 'ignore':
filepath = os.path.normpath(os.path.join(modpath, filename))
logger.debug('Extracting messages from %s', filepath)
extracted = extract_from_dir(modpath,
method_map,
options_map,
callback=log_callback)
for filename, lineno, message, comments, context in extracted:
catalog.add(message,
None, [(filename, lineno)],
auto_comments=comments,
context=context)
logger.info("%d messages extracted from component [%s]", len(catalog),
cident)
locale_path = Path(module.__path__[0]) / 'locale'
if locale_path.is_dir():
outfn = str(locale_path / '.pot')
else:
outfn = resource_filename(args.package, 'locale/%s.pot' % cident)
with io.open(outfn, 'wb') as outfd:
write_po(outfd, catalog, ignore_obsolete=True)
def run_gettext(dirname, for_js):
catalog = Catalog()
for filename, lineno, message, comments, context in extract_from_dir(
dirname,
method_map=[('**.js', 'javascript')] if for_js else [('**.py',
'python')]):
catalog.add(message,
None, [(filename, lineno)],
auto_comments=comments,
context=context)
sio = cStringIO.StringIO()
write_po(sio, catalog)
return sio.getvalue()
def main():
print "Extracting messages"
root = path.abspath(path.join(path.dirname(__file__), '..'))
catalog = Catalog(msgid_bugs_address=BUGS_ADDRESS,
copyright_holder=COPYRIGHT,
charset="utf-8")
def callback(filename, method, options):
if method != "ignore":
print strip_path(filename, root)
extracted_py = extract_from_dir(root, PY_METHODS, {}, PY_KEYWORDS,
COMMENT_TAGS, callback=callback,
strip_comment_tags=True)
extracted_js = extract_from_dir(root, [("static/js/**.js", "javascript")],
{}, JS_KEYWORDS,
COMMENT_TAGS, callback=callback,
strip_comment_tags=True)
for filename, lineno, message, comments in chain(extracted_py,
extracted_js):
catalog.add(message, None, [(strip_path(filename, root), lineno)],
auto_comments=comments)
output_path = path.join(root, "i18n")
if not path.isdir(output_path):
makedirs(output_path)
f = file(path.join(output_path, "messages.pot"), "w")
try:
write_po(f, catalog, width=79)
finally:
f.close()
print "All done."
def run(self):
mappings = self._get_mappings()
outfile = open(self.output_file, 'w')
try:
catalog = Catalog(project=self.distribution.get_name(),
version=self.distribution.get_version(),
msgid_bugs_address=self.msgid_bugs_address,
copyright_holder=self.copyright_holder,
charset=self.charset)
for dirname, (method_map, options_map) in mappings.items():
def callback(filename, method, options):
if method == 'ignore':
return
filepath = os.path.normpath(os.path.join(
dirname, filename))
optstr = ''
if options:
optstr = ' (%s)' % ', '.join(
['%s="%s"' % (k, v) for k, v in options.items()])
log.info('extracting messages from %s%s', filepath, optstr)
extracted = extract_from_dir(
dirname,
method_map,
options_map,
keywords=self._keywords,
comment_tags=self._add_comments,
callback=callback,
strip_comment_tags=self.strip_comments)
for filename, lineno, message, comments in extracted:
filepath = os.path.normpath(os.path.join(
dirname, filename))
catalog.add(message,
None, [(filepath, lineno)],
auto_comments=comments)
log.info('writing PO template file to %s' % self.output_file)
write_po(outfile,
catalog,
width=self.width,
no_location=self.no_location,
omit_header=self.omit_header,
sort_output=self.sort_output,
sort_by_file=self.sort_by_file)
finally:
outfile.close()
def build_pot(self, force=False):
"""
Extract translation strings and create Portable Object Template (POT)
from enabled source directories using defined extract rules.
Note:
May only work on internal '_pot' to return without touching
'self._pot'.
Keyword Arguments:
force (boolean): Default behavior is to proceed only if POT file
does not allready exists except if this argument is ``True``.
Returns:
babel.messages.catalog.Catalog: Catalog template object.
"""
if force or not self.check_template_path():
self.logger.info(("Proceeding to extraction to update the "
"template catalog (POT)"))
self._pot = Catalog(project=self.settings.SITE_NAME,
header_comment=self.header_comment)
# Follow all paths to search for pattern to extract
for extract_path in self.settings.I18N_EXTRACT_SOURCES:
msg = "Searching for pattern to extract in : {0}"
self.logger.debug(msg.format(extract_path))
extracted = extract_from_dir(
dirname=extract_path,
method_map=self.settings.I18N_EXTRACT_MAP,
options_map=self.settings.I18N_EXTRACT_OPTIONS,
)
# Proceed to extract from given path
for filename, lineno, message, comments, context in extracted:
filepath = os.path.normpath(
os.path.join(
os.path.basename(self.settings.SOURCES_DIR),
filename))
self._pot.add(
message,
None,
[(filepath, lineno)],
auto_comments=comments,
context=context,
)
with io.open(self.get_template_path(), "wb") as fp:
write_po(fp, self._pot)
return self._pot
def set_nereid(self):
"""
There are messages within the tryton code used in flash messages,
returned responses etc. This is spread over the codebase and this
function extracts the translation strings from code of installed
modules.
"""
pool = Pool()
Translation = pool.get('ir.translation')
to_create = []
for module, directory in self._get_installed_module_directories():
# skip messages from test files
if 'tests' in directory:
continue
for (filename, lineno, messages, comments, context) in \
extract_from_dir(directory,):
if isinstance(messages, basestring):
# messages could be a tuple if the function is ngettext
# where the messages for singular and plural are given as
# a tuple.
#
# So convert basestrings to tuples
messages = (messages, )
for message in messages:
translations = Translation.search([
('lang', '=', 'en_US'),
('type', '=', 'nereid'),
('name', '=', filename),
('src', '=', message),
('module', '=', module),
], limit=1)
if translations:
continue
to_create.append({
'name': filename,
'res_id': lineno,
'lang': 'en_US',
'src': message,
'type': 'nereid',
'module': module,
'comments': comments and '\n'.join(comments) or None,
})
if to_create:
Translation.create(to_create)
def build_pot(self, force=False):
"""
Extract translation strings and create Portable Object Template (POT)
from enabled source directories using defined extract rules.
Note:
May only work on internal '_pot' to return without touching
'self._pot'.
Keyword Arguments:
force (boolean): Default behavior is to proceed only if POT file
does not allready exists except if this argument is ``True``.
Returns:
babel.messages.catalog.Catalog: Catalog template object.
"""
if force or not self.check_template_path():
self.logger.info(("Proceeding to extraction to update the "
"template catalog (POT)"))
self._pot = Catalog(project=self.settings.SITE_NAME,
header_comment=self.header_comment)
# Follow all paths to search for pattern to extract
for extract_path in self.settings.I18N_EXTRACT_SOURCES:
msg = "Searching for pattern to extract in : {0}"
self.logger.debug(msg.format(extract_path))
extracted = extract_from_dir(
dirname=extract_path,
method_map=self.settings.I18N_EXTRACT_MAP,
options_map=self.settings.I18N_EXTRACT_OPTIONS
)
# Proceed to extract from given path
for filename, lineno, message, comments, context in extracted:
filepath = os.path.normpath(
os.path.join(
os.path.basename(self.settings.SOURCES_DIR),
filename
)
)
self._pot.add(message, None, [(filepath, lineno)],
auto_comments=comments, context=context)
with io.open(self.get_template_path(), 'wb') as fp:
write_po(fp, self._pot)
return self._pot
def extract_python_strings(dirname, outfile, domain=None):
"""Extract translatable strings from all Python files in `dir`.
Writes a PO template to `outfile`. Recognises `_` and `l_`. Needs babel!
"""
from babel.messages.catalog import Catalog
from babel.messages.extract import extract_from_dir
from babel.messages.pofile import write_po
base_dir = os.path.abspath(os.path.dirname(outfile))
msgs = extract_from_dir(dirname, keywords={'_': None, 'l_': None},
comment_tags=['l10n:'])
cat = Catalog(domain=domain, charset='utf-8')
for fname, lineno, message, comments, context in msgs:
filepath = os.path.join(dirname, fname)
cat.add(message, None, [(os.path.relpath(filepath, base_dir), lineno)],
auto_comments=comments, context=context)
with open(outfile, 'wb') as f:
write_po(f, cat)
def set_wtforms(self):
"""
There are some messages in WTForms which are provided by the framework,
namely default validator messages and errors occuring during the
processing (data coercion) stage. For example, in the case of the
IntegerField, if someone entered a value which was not valid as
an integer, then a message like “Not a valid integer value” would be
displayed.
"""
pool = Pool()
Translation = pool.get('ir.translation')
to_create = []
for (filename, lineno, messages, comments, context) in \
extract_from_dir(os.path.dirname(wtforms.__file__)):
if isinstance(messages, basestring):
# messages could be a tuple if the function is ngettext
# where the messages for singular and plural are given as
# a tuple.
#
# So convert basestrings to tuples
messages = (messages, )
for message in messages:
translations = Translation.search([
('lang', '=', 'en_US'),
('type', '=', 'wtforms'),
('name', '=', filename),
('src', '=', message),
('module', '=', 'nereid'),
], limit=1)
if translations:
continue
to_create.append({
'name': filename,
'res_id': lineno,
'lang': 'en_US',
'src': message,
'type': 'wtforms',
'module': 'nereid',
'comments': comments and '\n'.join(comments) or None,
})
if to_create:
Translation.create(to_create)
def extract(self, force=False):
"""
Extract translation strings from sources directory with extract rules then
create the template catalog with finded translation strings
Only proceed if the template catalog does not exists yet or if
``force`` argument is ``True`` (this will overwrite previous existing
POT file)
TODO: actually from the CLI usage this only update POT file when he does not
exist, else it keeps untouched, even if there changes or adds in translations
"""
if force or not self.check_template_path():
self.logger.info(
'Proceeding to extraction to update the template catalog (POT)'
)
self._catalog_template = Catalog(
project=self.settings.SITE_NAME,
header_comment=self.header_comment)
# Follow all paths to search for pattern to extract
for extract_path in self.settings.I18N_EXTRACT_SOURCES:
self.logger.debug(
'Searching for pattern to extract in : {0}'.format(
extract_path))
extracted = extract_from_dir(
dirname=extract_path,
method_map=self.settings.I18N_EXTRACT_MAP,
options_map=self.settings.I18N_EXTRACT_OPTIONS)
# Proceed to extract from given path
for filename, lineno, message, comments, context in extracted:
filepath = os.path.normpath(
os.path.join(
os.path.basename(self.settings.SOURCES_DIR),
filename))
self._catalog_template.add(message,
None, [(filepath, lineno)],
auto_comments=comments,
context=context)
outfile = open(self.get_template_path(), 'wb')
write_po(outfile, self._catalog_template)
outfile.close()
return self._catalog_template
def handle(self, *args, **options):
domains = options.get('domain')
if domains == "all":
domains = settings.DOMAIN_METHODS.keys()
else:
domains = [domains]
root = settings.ROOT
def callback(filename, method, options):
if method != 'ignore':
print " %s" % filename
for domain in domains:
print "Extracting all strings in domain %s..." % (domain)
methods = settings.DOMAIN_METHODS[domain]
extracted = extract_from_dir(root,
method_map=methods,
keywords=DEFAULT_KEYWORDS,
comment_tags=COMMENT_TAGS,
callback=callback,
options_map=OPTIONS_MAP,
)
catalog = create_pofile_from_babel(extracted)
catalog.savefile(os.path.join(root, 'locale', 'z-%s.pot' % domain))
if len(domains) > 1:
print "Concatenating all domains..."
pot_files = []
for i in domains:
pot_files.append(os.path.join(root, 'locale', 'z-%s.pot' % i))
z_keys = open(os.path.join(root, 'locale', 'z-keys.pot'), 'w+t')
z_keys.truncate()
command = ["msgcat"] + pot_files
p1 = Popen(command, stdout=z_keys)
p1.communicate()
z_keys.close()
for i in domains:
os.remove(os.path.join(root, 'locale', 'z-%s.pot' % i))
print 'done'
def extract_messages( # noqa
dirname, project=app.name, version=app.cfg.get('VERSION', ''),
charset='utf-8', domain=self.cfg.domain, locale=self.cfg.default_locale):
"""Extract messages from source code.
:param charset: charset to use in the output
:param domain: set domain name for locales
:param project: set project name in output
:param version: set project version in output
"""
Locale.parse(locale)
if not os.path.isdir(dirname):
raise SystemExit('%r is not a directory' % dirname)
catalog = Catalog(locale=locale, project=project, version=version, charset=charset)
for filename, lineno, message, comments, context in extract_from_dir(
dirname, method_map=self.cfg.sources_map, options_map=self.cfg.options_map):
filepath = os.path.normpath(os.path.join(dirname, filename))
catalog.add(message, None, [(filepath, lineno)],
auto_comments=comments, context=context)
locales_dir = self.cfg.locales_dirs[0]
output = os.path.join(locales_dir, locale, 'LC_MESSAGES', '%s.po' % domain)
if os.path.exists(output):
with open(output, 'rb') as f:
template = read_po(f, locale=locale, charset=charset)
template.update(catalog)
catalog = template
if not os.path.exists(os.path.dirname(output)):
os.makedirs(os.path.dirname(output))
logger.info('writing PO template file to %s', output)
outfile = open(output, 'wb')
try:
write_po(outfile, catalog, include_previous=True)
finally:
outfile.close()
def run(self):
mappings = self._get_mappings()
outfile = open(self.output_file, 'wb')
try:
catalog = Catalog(project=self.distribution.get_name(),
version=self.distribution.get_version(),
msgid_bugs_address=self.msgid_bugs_address,
copyright_holder=self.copyright_holder,
charset=self.charset)
for dirname, (method_map, options_map) in mappings.items():
def callback(filename, method, options):
if method == 'ignore':
return
filepath = os.path.normpath(os.path.join(dirname, filename))
optstr = ''
if options:
optstr = ' (%s)' % ', '.join(['%s="%s"' % (k, v) for
k, v in options.items()])
log.info('extracting messages from %s%s', filepath, optstr)
extracted = extract_from_dir(dirname, method_map, options_map,
keywords=self._keywords,
comment_tags=self._add_comments,
callback=callback,
strip_comment_tags=
self.strip_comments)
for filename, lineno, message, comments, context in extracted:
filepath = os.path.normpath(os.path.join(dirname, filename))
catalog.add(message, None, [(filepath, lineno)],
auto_comments=comments, context=context)
log.info('writing PO template file to %s' % self.output_file)
write_po(outfile, catalog, width=self.width,
no_location=self.no_location,
omit_header=self.omit_header,
sort_output=self.sort_output,
sort_by_file=self.sort_by_file)
finally:
outfile.close()
def extract_to_pot_catalog() -> Catalog:
mappings = [(ROOT_DIR, [("**.py", _extract_python)], {})]
pot_catalog = Catalog(default_locale_id)
for path, method_map, options_map in mappings:
def callback(filename, method, options):
if method == "ignore":
return
if os.path.isfile(path):
current_dir = os.getcwd()
extracted = check_and_call_extract_file(path, method_map,
options_map, callback, {},
[], False, current_dir)
else:
extracted = extract_from_dir(
path,
method_map,
options_map,
keywords={},
comment_tags=[],
callback=callback,
strip_comment_tags=False,
)
for filename, lineno, message, comments, context in extracted:
if os.path.isfile(path):
filepath = filename # already normalized
else:
filepath = os.path.normpath(os.path.join(path, filename))
pot_catalog.add(
message,
None,
[(os.path.relpath(filepath, ROOT_DIR), lineno)],
auto_comments=comments,
context=context,
)
return pot_catalog
def extract(self, force=False):
"""
Extract translation strings from sources directory with extract rules then
create the template catalog with finded translation strings
Only proceed if the template catalog does not exists yet or if
``force`` argument is ``True`` (this will overwrite previous existing
POT file)
"""
if force or not self.check_template_path():
self.logger.warning('Template catalog (POT) does not exists, extracting it')
self._catalog_template = Catalog(project=self.settings.SITE_NAME, header_comment=self.header_comment)
extracted = extract_from_dir(dirname=self.settings.SOURCES_DIR, method_map=self.settings.I18N_EXTRACT_MAP, options_map=self.settings.I18N_EXTRACT_OPTIONS)
for filename, lineno, message, comments, context in extracted:
filepath = os.path.normpath(os.path.join(os.path.basename(self.settings.SOURCES_DIR), filename))
self._catalog_template.add(message, None, [(filepath, lineno)], auto_comments=comments, context=context)
outfile = open(self.get_template_path(), 'wb')
write_po(outfile, self._catalog_template)
outfile.close()
return self._catalog_template
def extract_python_strings(dirname, outfile, domain=None):
"""Extract translatable strings from all Python files in `dir`.
Writes a PO template to `outfile`. Recognises `_` and `l_`. Needs babel!
"""
from babel.messages.catalog import Catalog
from babel.messages.extract import extract_from_dir
from babel.messages.pofile import write_po
base_dir = os.path.abspath(os.path.dirname(outfile))
msgs = extract_from_dir(dirname,
keywords={
'_': None,
'l_': None
},
comment_tags=['l10n:'])
cat = Catalog(domain=domain, charset='utf-8')
for fname, lineno, message, comments, context in msgs:
filepath = os.path.join(dirname, fname)
cat.add(message,
None, [(os.path.relpath(filepath, base_dir), lineno)],
auto_comments=comments,
context=context)
with open(outfile, 'wb') as f:
write_po(f, cat)
def do_extract_messages(target=('t', ''),
domain=('d', 'messages'),
i18n_dir=('i', ''),
all=('a', False)):
"""
Extract messages and create pot file.
"""
if not domain in ('messages', 'jsmessages'):
print_status('invalid domain.')
sys.exit(1)
if not target and not all:
print_status('Please specify target.')
sys.exit(1)
elif target == 'kay':
print_status('Extracting core strings')
root = kay.KAY_DIR
elif all:
targets = get_user_apps()
for target in targets:
do_extract_messages(target=target,
domain=domain,
i18n_dir=None,
all=False)
for template_dir in settings.TEMPLATE_DIRS:
do_extract_messages(target=template_dir,
domain=domain,
i18n_dir=settings.I18N_DIR,
all=False)
sys.exit(0)
else:
root = path.abspath(target)
if not path.isdir(root):
print_status('source folder missing')
sys.exit(1)
print_status('Extracting from %s' % root)
if domain == 'messages':
methods = METHODS
else:
methods = JSMETHODS
catalog = Catalog(msgid_bugs_address=BUGS_ADDRESS,
copyright_holder=COPYRIGHT,
charset='utf-8')
def callback(filename, method, options):
if method != 'ignore':
print_status(strip_path(filename, root))
option = {}
option['extensions'] = ','.join(settings.JINJA2_EXTENSIONS)
option.update(settings.JINJA2_ENVIRONMENT_KWARGS)
options = {
'**/templates/**.*': option,
'**.html': option,
}
extracted = extract_from_dir(root,
methods,
options,
KEYWORDS,
COMMENT_TAGS,
callback=callback,
strip_comment_tags=True)
for filename, lineno, message, comments in extracted:
catalog.add(message,
None, [(strip_path(filename, root), lineno)],
auto_comments=comments)
if not i18n_dir:
i18n_dir = path.join(root, 'i18n')
if not path.isdir(i18n_dir):
makedirs(i18n_dir)
f = file(path.join(i18n_dir, domain + '.pot'), 'w')
try:
write_po(f, catalog, width=79)
finally:
f.close()
print_status('All done.')
def extract(self, argv):
"""Subcommand for extracting messages from source files and generating
a POT file.
:param argv: the command arguments
"""
parser = OptionParser(usage=self.usage % ('extract', 'dir1 <dir2> ...'),
description=self.commands['extract'])
parser.add_option('--charset', dest='charset',
help='charset to use in the output (default '
'"%default")')
parser.add_option('-k', '--keyword', dest='keywords', action='append',
help='keywords to look for in addition to the '
'defaults. You can specify multiple -k flags on '
'the command line.')
parser.add_option('--no-default-keywords', dest='no_default_keywords',
action='store_true',
help="do not include the default keywords")
parser.add_option('--mapping', '-F', dest='mapping_file',
help='path to the extraction mapping file')
parser.add_option('--no-location', dest='no_location',
action='store_true',
help='do not include location comments with filename '
'and line number')
parser.add_option('--omit-header', dest='omit_header',
action='store_true',
help='do not include msgid "" entry in header')
parser.add_option('-o', '--output', dest='output',
help='path to the output POT file')
parser.add_option('-w', '--width', dest='width', type='int',
help="set output line width (default 76)")
parser.add_option('--no-wrap', dest='no_wrap', action = 'store_true',
help='do not break long message lines, longer than '
'the output line width, into several lines')
parser.add_option('--sort-output', dest='sort_output',
action='store_true',
help='generate sorted output (default False)')
parser.add_option('--sort-by-file', dest='sort_by_file',
action='store_true',
help='sort output by file location (default False)')
parser.add_option('--msgid-bugs-address', dest='msgid_bugs_address',
metavar='EMAIL@ADDRESS',
help='set report address for msgid')
parser.add_option('--copyright-holder', dest='copyright_holder',
help='set copyright holder in output')
parser.add_option('--project', dest='project',
help='set project name in output')
parser.add_option('--version', dest='version',
help='set project version in output')
parser.add_option('--add-comments', '-c', dest='comment_tags',
metavar='TAG', action='append',
help='place comment block with TAG (or those '
'preceding keyword lines) in output file. One '
'TAG per argument call')
parser.add_option('--strip-comment-tags', '-s',
dest='strip_comment_tags', action='store_true',
help='Strip the comment tags from the comments.')
parser.set_defaults(charset='utf-8', keywords=[],
no_default_keywords=False, no_location=False,
omit_header = False, width=None, no_wrap=False,
sort_output=False, sort_by_file=False,
comment_tags=[], strip_comment_tags=False)
options, args = parser.parse_args(argv)
if not args:
parser.error('incorrect number of arguments')
if options.output not in (None, '-'):
outfile = open(options.output, 'wb')
else:
outfile = sys.stdout
keywords = DEFAULT_KEYWORDS.copy()
if options.no_default_keywords:
if not options.keywords:
parser.error('you must specify new keywords if you disable the '
'default ones')
keywords = {}
if options.keywords:
keywords.update(parse_keywords(options.keywords))
if options.mapping_file:
fileobj = open(options.mapping_file, 'U')
try:
method_map, options_map = parse_mapping(fileobj)
finally:
fileobj.close()
else:
method_map = DEFAULT_MAPPING
options_map = {}
if options.width and options.no_wrap:
parser.error("'--no-wrap' and '--width' are mutually exclusive.")
elif not options.width and not options.no_wrap:
options.width = 76
if options.sort_output and options.sort_by_file:
parser.error("'--sort-output' and '--sort-by-file' are mutually "
"exclusive")
try:
catalog = Catalog(project=options.project,
version=options.version,
msgid_bugs_address=options.msgid_bugs_address,
copyright_holder=options.copyright_holder,
charset=options.charset)
for dirname in args:
if not os.path.isdir(dirname):
parser.error('%r is not a directory' % dirname)
def callback(filename, method, options):
if method == 'ignore':
return
filepath = os.path.normpath(os.path.join(dirname, filename))
optstr = ''
if options:
optstr = ' (%s)' % ', '.join(['%s="%s"' % (k, v) for
k, v in options.items()])
self.log.info('extracting messages from %s%s', filepath,
optstr)
extracted = extract_from_dir(dirname, method_map, options_map,
keywords, options.comment_tags,
callback=callback,
strip_comment_tags=
options.strip_comment_tags)
for filename, lineno, message, comments, context in extracted:
filepath = os.path.normpath(os.path.join(dirname, filename))
catalog.add(message, None, [(filepath, lineno)],
auto_comments=comments, context=context)
if options.output not in (None, '-'):
self.log.info('writing PO template file to %s' % options.output)
write_po(outfile, catalog, width=options.width,
no_location=options.no_location,
omit_header=options.omit_header,
sort_output=options.sort_output,
sort_by_file=options.sort_by_file)
finally:
if options.output:
outfile.close()
def get_translations(quiet=True):
''' Extract translations put in db and create mo file'''
sql = "UPDATE translation SET active=0 WHERE lang='en';"
engine.execute(sql)
method_map = [
('**/templates/**.html', 'jinja2'),
('**/themes/**.html', 'jinja2'),
('**.py', 'python')
]
if not quiet:
print 'Extracting translations'
extracted = extract.extract_from_dir('.', method_map=method_map)
DIR = os.path.dirname(os.path.realpath(__file__))
catalog = Catalog(project='hrd')
for filename, lineno, message, comments, context in extracted:
filepath = os.path.normpath(os.path.join(DIR, filename))
catalog.add(
message, None, [(filepath, lineno)],
auto_comments=comments, context=context
)
if isinstance(message, tuple):
values = message
else:
values = (message, '')
sql = """
SELECT active FROM translation
WHERE
lang = 'en' and id=? and plural=?;
"""
result = conn.execute(sql, values).first()
if result is None:
sql = """
INSERT INTO translation (id, plural, lang, active)
VALUES (?, ?, 'en', 1);
"""
conn.execute(sql, values)
elif result[0] == 0:
sql = """
UPDATE translation
SET active = 1
WHERE
lang = 'en' and id=? and plural=?;
"""
conn.execute(sql, values)
path = os.path.join(DIR, 'translations')
try:
os.makedirs(path)
except OSError:
pass
outfile = open(os.path.join(path, 'messages.pot'), 'wb')
try:
if not quiet:
print 'writing POT template file to trans'
write_po(outfile, catalog)
finally:
outfile.close()
def get_translations(quiet=True):
''' Extract translations put in db and create mo file'''
sql = "UPDATE translation SET active=0 WHERE lang='en';"
engine.execute(sql)
method_map = [('**/templates/**.html', 'jinja2'),
('**/themes/**.html', 'jinja2'), ('**.py', 'python')]
if not quiet:
print 'Extracting translations'
extracted = extract.extract_from_dir('.', method_map=method_map)
DIR = os.path.dirname(os.path.realpath(__file__))
catalog = Catalog(project='hrd')
for filename, lineno, message, comments, context in extracted:
filepath = os.path.normpath(os.path.join(DIR, filename))
catalog.add(message,
None, [(filepath, lineno)],
auto_comments=comments,
context=context)
if isinstance(message, tuple):
values = message
else:
values = (message, '')
sql = """
SELECT active FROM translation
WHERE
lang = 'en' and id=? and plural=?;
"""
result = conn.execute(sql, values).first()
if result is None:
sql = """
INSERT INTO translation (id, plural, lang, active)
VALUES (?, ?, 'en', 1);
"""
conn.execute(sql, values)
elif result[0] == 0:
sql = """
UPDATE translation
SET active = 1
WHERE
lang = 'en' and id=? and plural=?;
"""
conn.execute(sql, values)
path = os.path.join(DIR, 'translations')
try:
os.makedirs(path)
except OSError:
pass
outfile = open(os.path.join(path, 'messages.pot'), 'wb')
try:
if not quiet:
print 'writing POT template file to trans'
write_po(outfile, catalog)
finally:
outfile.close()
return
filepath = os.path.normpath(os.path.join(dirname, filename))
optstr = ''
if options:
optstr = ' (%s)' % ', '.join(['%s="%s"' % (k, v) for
k, v in options.items()])
print ('extracting messages from %s%s' %( filepath, optstr))
outfile = open(output_file, 'w')
try:
catalog = Catalog(project=project,
version=version,
charset="UTF-8")
extracted = extract_from_dir(dirname, method_map, options_map,
keywords=keywords,
callback=callback)
for filename, lineno, message, comments in extracted:
filepath = os.path.normpath(os.path.join(dirname, filename))
catalog.add(message, None, [(filepath, lineno)],
auto_comments=comments)
print ('writing PO template file to %s' % output_file)
write_po(outfile, catalog, sort_by_file = True)
finally:
outfile.close()
def extract_command(outputdir, domain_methods, text_domain, keywords,
comment_tags, base_dir, project, version,
msgid_bugs_address):
"""Extracts strings into .pot files
:arg domain: domains to generate strings for or 'all' for all domains
:arg outputdir: output dir for .pot files; usually
locale/templates/LC_MESSAGES/
:arg domain_methods: DOMAIN_METHODS setting
:arg text_domain: TEXT_DOMAIN settings
:arg keywords: KEYWORDS setting
:arg comment_tags: COMMENT_TAGS setting
:arg base_dir: BASE_DIR setting
:arg project: PROJECT setting
:arg version: VERSION setting
:arg msgid_bugs_address: MSGID_BUGS_ADDRESS setting
"""
# Must monkeypatch first to fix i18n extensions stomping issues!
monkeypatch_i18n()
# Create the outputdir if it doesn't exist
outputdir = os.path.abspath(outputdir)
if not os.path.isdir(outputdir):
print('Creating output dir %s ...' % outputdir)
os.makedirs(outputdir)
domains = domain_methods.keys()
def callback(filename, method, options):
if method != 'ignore':
print(' %s' % filename)
# Extract string for each domain
for domain in domains:
print('Extracting all strings in domain %s...' % domain)
methods = domain_methods[domain]
catalog = Catalog(
header_comment='',
project=project,
version=version,
msgid_bugs_address=msgid_bugs_address,
charset='utf-8',
)
extracted = extract_from_dir(
base_dir,
method_map=methods,
options_map=generate_options_map(),
keywords=keywords,
comment_tags=comment_tags,
callback=callback,
)
for filename, lineno, msg, cmts, ctxt in extracted:
catalog.add(msg,
None, [(filename, lineno)],
auto_comments=cmts,
context=ctxt)
with open(os.path.join(outputdir, '%s.pot' % domain), 'wb') as fp:
write_po(fp, catalog, width=80)
print('Done')
def handle(self, *args, **options):
domains = options.get('domain')
outputdir = os.path.abspath(options.get('outputdir'))
if not os.path.isdir(outputdir):
if not options.get('create'):
print ('Output directory must exist (%s) unless -c option is '
'given. Specify one with --output-dir' % outputdir)
return 'FAILURE\n'
os.makedirs(outputdir)
if domains == 'all':
domains = settings.DOMAIN_METHODS.keys()
else:
domains = [domains]
root = settings.ROOT
def callback(filename, method, options):
if method != 'ignore':
print ' %s' % filename
for domain in domains:
print 'Extracting all strings in domain %s...' % (domain)
methods = settings.DOMAIN_METHODS[domain]
extracted = extract_from_dir(root,
method_map=methods,
keywords=TOWER_KEYWORDS,
comment_tags=COMMENT_TAGS,
callback=callback,
options_map=OPTIONS_MAP,
)
catalog = create_pofile_from_babel(extracted)
if not os.path.exists(outputdir):
raise Exception('Expected %s to exist... BAILING' % outputdir)
catalog.savefile(os.path.join(outputdir, '%s.pot' % domain))
pot_files = []
for i in [x for x in domains if x not in standalone_domains]:
pot_files.append(os.path.join(outputdir, '%s.pot' % i))
if len(pot_files) > 1:
print ('Concatenating the non-standalone domains into %s.pot' %
TEXT_DOMAIN)
final_out = os.path.join(outputdir, '%s.pot' % TEXT_DOMAIN)
# We add final_out back on because msgcat will combine all
# specified files. We'll redirect everything back in to
# final_out in a minute.
pot_files.append(final_out)
meltingpot = tempfile.TemporaryFile()
command = ['msgcat'] + pot_files
p1 = Popen(command, stdout=meltingpot)
p1.communicate()
meltingpot.seek(0)
# w+ truncates the file first
with open(final_out, 'w+') as final:
final.write(meltingpot.read())
meltingpot.close()
for i in [x for x in domains if x not in standalone_domains]:
os.remove(os.path.join(outputdir, '%s.pot' % i))
print 'Done'
def extract_command(outputdir, domain_methods, text_domain, keywords,
comment_tags, base_dir, project, version,
msgid_bugs_address):
"""Extracts strings into .pot files
:arg domain: domains to generate strings for or 'all' for all domains
:arg outputdir: output dir for .pot files; usually
locale/templates/LC_MESSAGES/
:arg domain_methods: DOMAIN_METHODS setting
:arg text_domain: TEXT_DOMAIN settings
:arg keywords: KEYWORDS setting
:arg comment_tags: COMMENT_TAGS setting
:arg base_dir: BASE_DIR setting
:arg project: PROJECT setting
:arg version: VERSION setting
:arg msgid_bugs_address: MSGID_BUGS_ADDRESS setting
"""
# Must monkeypatch first to fix i18n extensions stomping issues!
monkeypatch_i18n()
# Create the outputdir if it doesn't exist
outputdir = os.path.abspath(outputdir)
if not os.path.isdir(outputdir):
print('Creating output dir %s ...' % outputdir)
os.makedirs(outputdir)
domains = domain_methods.keys()
def callback(filename, method, options):
if method != 'ignore':
print(' %s' % filename)
# Extract string for each domain
for domain in domains:
print('Extracting all strings in domain %s...' % domain)
methods = domain_methods[domain]
catalog = Catalog(
header_comment='',
project=project,
version=version,
msgid_bugs_address=msgid_bugs_address,
charset='utf-8',
)
extracted = extract_from_dir(
base_dir,
method_map=methods,
options_map=generate_options_map(),
keywords=keywords,
comment_tags=comment_tags,
callback=callback,
)
for filename, lineno, msg, cmts, ctxt in extracted:
catalog.add(msg, None, [(filename, lineno)], auto_comments=cmts,
context=ctxt)
with open(os.path.join(outputdir, '%s.pot' % domain), 'wb') as fp:
write_po(fp, catalog, width=80)
print('Done')
def handle(self, *args, **options):
# Must monkeypatch first to fix InternationalizationExtension
# stomping issues! See docstring for details.
monkeypatch_i18n()
# Get all the settings we need; we uppercase them so they're
# obviously setings
DOMAIN_METHODS = get_setting("DOMAIN_METHODS")
STANDALONE_DOMAINS = get_setting("STANDALONE_DOMAINS")
KEYWORDS = get_setting("KEYWORDS")
COMMENT_TAGS = get_setting("COMMENT_TAGS")
ROOT = get_setting("ROOT")
keywords_dict = dict([(keyword, None) for keyword in KEYWORDS])
domains = options.get("domain")
outputdir = os.path.abspath(options.get("outputdir"))
if not os.path.isdir(outputdir):
if not options.get("create"):
print (
"Output directory must exist (%s) unless -c option is "
"given. Specify one with --output-dir" % outputdir
)
# FIXME: This should return a non-zero exit code and
# print to stderr however that works in Django.
return "FAILURE\n"
os.makedirs(outputdir)
if domains == DEFAULT_DOMAIN_VALUE:
domains = DOMAIN_METHODS.keys()
else:
domains = [domains]
def callback(filename, method, options):
if method != "ignore":
print " %s" % filename
for domain in domains:
print "Extracting all strings in domain %s..." % (domain)
methods = DOMAIN_METHODS[domain]
extracted = extract_from_dir(
ROOT,
method_map=methods,
keywords=keywords_dict,
comment_tags=COMMENT_TAGS,
callback=callback,
options_map=generate_options_map(),
)
catalog = create_pofile_from_babel(extracted)
if not os.path.exists(outputdir):
# FIXME: This should return a non-zero exit code and
# print to stderr and be consistent
raise Exception("Expected %s to exist... BAILING" % outputdir)
catalog.savefile(os.path.join(outputdir, "%s.pot" % domain))
not_standalone_domains = [dom for dom in domains if dom not in STANDALONE_DOMAINS]
pot_files = []
for dom in not_standalone_domains:
pot_files.append(os.path.join(outputdir, "%s.pot" % dom))
if len(pot_files) > 1:
pot_file = get_setting("TEXT_DOMAIN") + ".pot"
print ("Concatenating the non-standalone domains into %s" % pot_file)
final_out = os.path.join(outputdir, pot_file)
# We add final_out back on because msgcat will combine all
# specified files. We'll redirect everything back in to
# final_out in a minute.
pot_files.append(final_out)
meltingpot = tempfile.TemporaryFile()
p1 = Popen(["msgcat"] + pot_files, stdout=meltingpot)
p1.communicate()
meltingpot.seek(0)
# w+ truncates the file first
with open(final_out, "w+") as final:
final.write(meltingpot.read())
meltingpot.close()
for dom in not_standalone_domains:
os.remove(os.path.join(outputdir, "%s.pot" % dom))
print "Done"
def extract_command(domain, outputdir, domain_methods, standalone_domains,
text_domain, keywords, comment_tags, base_dir):
"""Extracts strings into .pot files
:arg domain: domains to generate strings for or 'all' for all domains
:arg outputdir: output dir for .pot files; usually
locale/templates/LC_MESSAGES/
:arg domain_methods: DOMAIN_METHODS setting
:arg standalone_domains: STANDALONE_DOMAINS setting
:arg text_domain: TEXT_DOMAIN settings
:arg keywords: KEYWORDS setting
:arg comment_tags: COMMENT_TAGS setting
:arg base_dir: BASE_DIR setting
"""
# Must monkeypatch first to fix i18n extensions stomping issues!
monkeypatch_i18n()
# Create the outputdir if it doesn't exist
outputdir = os.path.abspath(outputdir)
if not os.path.isdir(outputdir):
print 'Creating output dir %s ...' % outputdir
os.makedirs(outputdir)
# Figure out what domains to extract
if domain == DEFAULT_DOMAIN_VALUE:
domains = domain_methods.keys()
else:
domains = [domain]
def callback(filename, method, options):
if method != 'ignore':
print ' %s' % filename
# Extract string for each domain
for domain in domains:
print 'Extracting all strings in domain %s...' % (domain)
methods = domain_methods[domain]
catalog = Catalog(charset='utf-8', header_comment='')
extracted = extract_from_dir(
base_dir,
method_map=methods,
options_map=generate_options_map(),
keywords=keywords,
comment_tags=comment_tags,
callback=callback,
)
for filename, lineno, msg, cmts, ctxt in extracted:
catalog.add(msg, None, [(filename, lineno)], auto_comments=cmts,
context=ctxt)
with open(os.path.join(outputdir, '%s.pot' % domain), 'w') as fp:
write_po(fp, catalog, width=80)
not_standalone_domains = [
dom for dom in domains
if dom not in standalone_domains
]
pot_files = []
for dom in not_standalone_domains:
pot_files.append(os.path.join(outputdir, '%s.pot' % dom))
if len(pot_files) > 1:
pot_file = text_domain + '.pot'
print ('Concatenating the non-standalone domains into %s' %
pot_file)
final_out = os.path.join(outputdir, pot_file)
# We add final_out back on because msgcat will combine all
# specified files. We'll redirect everything back in to
# final_out in a minute.
pot_files.append(final_out)
meltingpot = tempfile.TemporaryFile()
p1 = Popen(['msgcat'] + pot_files, stdout=meltingpot)
p1.communicate()
meltingpot.seek(0)
# w+ truncates the file first
with open(final_out, 'w+') as final:
final.write(meltingpot.read())
meltingpot.close()
for dom in not_standalone_domains:
os.remove(os.path.join(outputdir, '%s.pot' % dom))
print 'Done'
output_file = "mb_server.pot"
def callback(filename, method, options):
if method == "ignore":
return
filepath = os.path.normpath(os.path.join(dirname, filename))
optstr = ""
if options:
optstr = " (%s)" % ", ".join(['%s="%s"' % (k, v) for k, v in options.items()])
print("extracting messages from %s%s" % (filepath, optstr))
outfile = open(output_file, "w")
try:
catalog = Catalog(project=project, version=version, charset="UTF-8")
extracted = extract_from_dir(dirname, method_map, options_map, keywords=keywords, callback=callback)
for filename, lineno, message, comments in extracted:
filepath = os.path.normpath(os.path.join(dirname, filename))
catalog.add(message, None, [(filepath, lineno)], auto_comments=comments)
print("writing PO template file to %s" % output_file)
write_po(outfile, catalog, sort_by_file=True)
finally:
outfile.close()
def do_extract_messages(target=('t', ''), domain=('d', 'messages'),
i18n_dir=('i', ''), all=('a', False)):
"""
Extract messages and create pot file.
"""
if not domain in ('messages', 'jsmessages'):
print_status('invalid domain.')
sys.exit(1)
if not target and not all:
print_status('Please specify target.')
sys.exit(1)
elif target == 'kay':
print_status('Extracting core strings')
root = kay.KAY_DIR
elif all:
targets = get_user_apps()
for target in targets:
do_extract_messages(target=target, domain=domain, i18n_dir=None,
all=False)
sys.exit(0)
else:
root = path.abspath(target)
if not path.isdir(root):
print_status('source folder missing')
sys.exit(1)
print_status('Extracting from %s' % root)
if domain == 'messages':
methods = METHODS
else:
methods = JSMETHODS
catalog = Catalog(msgid_bugs_address=BUGS_ADDRESS,
copyright_holder=COPYRIGHT, charset='utf-8')
def callback(filename, method, options):
if method != 'ignore':
print_status(strip_path(filename, root))
option = {}
option['extensions'] = ','.join(settings.JINJA2_EXTENSIONS)
option.update(settings.JINJA2_ENVIRONMENT_KWARGS)
options = {
'**/templates/**.*': option,
'**.html': option,
}
extracted = extract_from_dir(root, methods, options, KEYWORDS,
COMMENT_TAGS, callback=callback,
strip_comment_tags=True)
for filename, lineno, message, comments in extracted:
catalog.add(message, None, [(strip_path(filename, root), lineno)],
auto_comments=comments)
if not i18n_dir:
i18n_dir = path.join(root, 'i18n')
if not path.isdir(i18n_dir):
makedirs(i18n_dir)
f = file(path.join(i18n_dir, domain+'.pot'), 'w')
try:
write_po(f, catalog, width=79)
finally:
f.close()
print_status('All done.')
def extract(self, argv):
"""Subcommand for extracting messages from source files and generating
a POT file.
:param argv: the command arguments
"""
parser = OptionParser(usage=self.usage % ('extract', 'dir1 <dir2> ...'),
description=self.commands['extract'])
parser.add_option('--charset', dest='charset',
help='charset to use in the output (default '
'"%default")')
parser.add_option('-k', '--keyword', dest='keywords', action='append',
help='keywords to look for in addition to the '
'defaults. You can specify multiple -k flags on '
'the command line.')
parser.add_option('--no-default-keywords', dest='no_default_keywords',
action='store_true',
help="do not include the default keywords")
parser.add_option('--mapping', '-F', dest='mapping_file',
help='path to the extraction mapping file')
parser.add_option('--no-location', dest='no_location',
action='store_true',
help='do not include location comments with filename '
'and line number')
parser.add_option('--omit-header', dest='omit_header',
action='store_true',
help='do not include msgid "" entry in header')
parser.add_option('-o', '--output', dest='output',
help='path to the output POT file')
parser.add_option('-w', '--width', dest='width', type='int',
help="set output line width (default 76)")
parser.add_option('--no-wrap', dest='no_wrap', action = 'store_true',
help='do not break long message lines, longer than '
'the output line width, into several lines')
parser.add_option('--sort-output', dest='sort_output',
action='store_true',
help='generate sorted output (default False)')
parser.add_option('--sort-by-file', dest='sort_by_file',
action='store_true',
help='sort output by file location (default False)')
parser.add_option('--msgid-bugs-address', dest='msgid_bugs_address',
metavar='EMAIL@ADDRESS',
help='set report address for msgid')
parser.add_option('--copyright-holder', dest='copyright_holder',
help='set copyright holder in output')
parser.add_option('--project', dest='project',
help='set project name in output')
parser.add_option('--version', dest='version',
help='set project version in output')
parser.add_option('--add-comments', '-c', dest='comment_tags',
metavar='TAG', action='append',
help='place comment block with TAG (or those '
'preceding keyword lines) in output file. One '
'TAG per argument call')
parser.add_option('--strip-comment-tags', '-s',
dest='strip_comment_tags', action='store_true',
help='Strip the comment tags from the comments.')
parser.set_defaults(charset='utf-8', keywords=[],
no_default_keywords=False, no_location=False,
omit_header = False, width=None, no_wrap=False,
sort_output=False, sort_by_file=False,
comment_tags=[], strip_comment_tags=False)
options, args = parser.parse_args(argv)
if not args:
parser.error('incorrect number of arguments')
if options.output not in (None, '-'):
outfile = open(options.output, 'wb')
else:
outfile = sys.stdout
keywords = DEFAULT_KEYWORDS.copy()
if options.no_default_keywords:
if not options.keywords:
parser.error('you must specify new keywords if you disable the '
'default ones')
keywords = {}
if options.keywords:
keywords.update(parse_keywords(options.keywords))
if options.mapping_file:
fileobj = open(options.mapping_file, 'U')
try:
method_map, options_map = parse_mapping(fileobj)
finally:
fileobj.close()
else:
method_map = DEFAULT_MAPPING
options_map = {}
if options.width and options.no_wrap:
parser.error("'--no-wrap' and '--width' are mutually exclusive.")
elif not options.width and not options.no_wrap:
options.width = 76
if options.sort_output and options.sort_by_file:
parser.error("'--sort-output' and '--sort-by-file' are mutually "
"exclusive")
try:
catalog = Catalog(project=options.project,
version=options.version,
msgid_bugs_address=options.msgid_bugs_address,
copyright_holder=options.copyright_holder,
charset=options.charset)
for dirname in args:
if not os.path.isdir(dirname):
parser.error('%r is not a directory' % dirname)
def callback(filename, method, options):
if method == 'ignore':
return
filepath = os.path.normpath(os.path.join(dirname, filename))
optstr = ''
if options:
optstr = ' (%s)' % ', '.join(['%s="%s"' % (k, v) for
k, v in options.items()])
self.log.info('extracting messages from %s%s', filepath,
optstr)
extracted = extract_from_dir(dirname, method_map, options_map,
keywords, options.comment_tags,
callback=callback,
strip_comment_tags=
options.strip_comment_tags)
for filename, lineno, message, comments in extracted:
filepath = os.path.normpath(os.path.join(dirname, filename))
catalog.add(message, None, [(filepath, lineno)],
auto_comments=comments)
if options.output not in (None, '-'):
self.log.info('writing PO template file to %s' % options.output)
write_po(outfile, catalog, width=options.width,
no_location=options.no_location,
omit_header=options.omit_header,
sort_output=options.sort_output,
sort_by_file=options.sort_by_file)
finally:
if options.output:
outfile.close()
def handle(self, *args, **options):
domains = options.get('domain')
outputdir = os.path.abspath(options.get('outputdir'))
if not os.path.isdir(outputdir):
if not options.get('create'):
print(
'Output directory must exist (%s) unless -c option is '
'given. Specify one with --output-dir' % outputdir)
return 'FAILURE\n'
os.makedirs(outputdir)
if domains == 'all':
domains = settings.DOMAIN_METHODS.keys()
else:
domains = [domains]
root = settings.ROOT
def callback(filename, method, options):
if method != 'ignore':
print ' %s' % filename
for domain in domains:
print 'Extracting all strings in domain %s...' % (domain)
methods = settings.DOMAIN_METHODS[domain]
extracted = extract_from_dir(
root,
method_map=methods,
keywords=KEYWORDS,
comment_tags=COMMENT_TAGS,
callback=callback,
options_map=OPTIONS_MAP,
)
catalog = create_pofile_from_babel(extracted)
if not os.path.exists(outputdir):
raise Exception('Expected %s to exist... BAILING' % outputdir)
catalog.savefile(os.path.join(outputdir, '%s.pot' % domain))
pot_files = []
for i in [x for x in domains if x not in STANDALONE_DOMAINS]:
pot_files.append(os.path.join(outputdir, '%s.pot' % i))
if len(pot_files) > 1:
print('Concatenating the non-standalone domains into %s.pot' %
TEXT_DOMAIN)
final_out = os.path.join(outputdir, '%s.pot' % TEXT_DOMAIN)
# We add final_out back on because msgcat will combine all
# specified files. We'll redirect everything back in to
# final_out in a minute.
pot_files.append(final_out)
meltingpot = tempfile.TemporaryFile()
p1 = Popen(['msgcat'] + pot_files, stdout=meltingpot)
p1.communicate()
meltingpot.seek(0)
# w+ truncates the file first
with open(final_out, 'w+') as final:
final.write(meltingpot.read())
meltingpot.close()
for i in [x for x in domains if x not in STANDALONE_DOMAINS]:
os.remove(os.path.join(outputdir, '%s.pot' % i))
print 'Done'