def update_translations(self, component, previous_head):
cmd = [
'msgmerge', '--update', 'FILE', component.get_new_base_filename()
]
for translation in component.translation_set.all():
cmd[2] = translation.get_filename()
popen_wrapper(cmd, OSError)
def update_translations(self, component, previous_head):
cmd = [
'msgmerge', '--update', 'FILE', component.get_new_base_filename()
]
for translation in component.translation_set.all():
cmd[2] = translation.get_filename()
popen_wrapper(cmd)
def write_po_file(self, potfile, locale):
"""
Creates or updates the PO file for self.domain and :param locale:.
Uses contents of the existing :param potfile:.
Uses mguniq, msgmerge, and msgattrib GNU gettext utilities.
"""
args = ["msguniq", "--to-code=utf-8"]
if self.wrap:
args.append(self.wrap)
if self.location:
args.append(self.location)
args.append(potfile)
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError("errors happened while running msguniq\n%s" % errors)
elif self.verbosity > 0:
self.stdout.write(errors)
basedir = os.path.join(os.path.dirname(potfile), locale, "LC_MESSAGES")
if not os.path.isdir(basedir):
os.makedirs(basedir)
pofile = os.path.join(basedir, "%s.po" % str(self.domain))
if os.path.exists(pofile):
with open(potfile, "w") as fp:
fp.write(msgs)
args = ["msgmerge", "-q"]
if self.wrap:
args.append(self.wrap)
if self.location:
args.append(self.location)
args.extend([pofile, potfile])
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError("errors happened while running msgmerge\n%s" % errors)
elif self.verbosity > 0:
self.stdout.write(errors)
elif not self.invoked_for_django:
msgs = self.copy_plural_forms(msgs, locale)
msgs = msgs.replace("#. #-#-#-#-# %s.pot (PACKAGE VERSION) #-#-#-#-#\n" % self.domain, "")
with open(pofile, "w") as fp:
fp.write(msgs)
if self.no_obsolete:
args = ["msgattrib", "-o", pofile, "--no-obsolete"]
if self.wrap:
args.append(self.wrap)
if self.location:
args.append(self.location)
args.append(pofile)
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError("errors happened while running msgattrib\n%s" % errors)
elif self.verbosity > 0:
self.stdout.write(errors)
def write_po_file(self, potfile, locale):
"""
Creates or updates the PO file for self.domain and :param locale:.
Uses contents of the existing :param potfile:.
Uses msgmerge, and msgattrib GNU gettext utilities.
"""
basedir = os.path.join(os.path.dirname(potfile), locale, 'LC_MESSAGES')
if not os.path.isdir(basedir):
os.makedirs(basedir)
pofile = os.path.join(basedir, '%s.po' % str(self.domain))
if os.path.exists(pofile):
args = ['msgmerge', '-q']
if self.wrap:
args.append(self.wrap)
if self.location:
args.append(self.location)
args.extend([pofile, potfile])
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError(
"errors happened while running msgmerge\n%s" % errors)
elif self.verbosity > 0:
self.stdout.write(errors)
else:
with open(potfile, 'r') as fp:
msgs = fp.read()
if not self.invoked_for_django:
msgs = self.copy_plural_forms(msgs, locale)
msgs = msgs.replace(
"#. #-#-#-#-# %s.pot (PACKAGE VERSION) #-#-#-#-#\n" %
self.domain, "")
with open(pofile, 'w') as fp:
fp.write(msgs)
if self.no_obsolete:
args = ['msgattrib', '-o', pofile, '--no-obsolete']
if self.wrap:
args.append(self.wrap)
if self.location:
args.append(self.location)
args.append(pofile)
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError(
"errors happened while running msgattrib\n%s" % errors)
elif self.verbosity > 0:
self.stdout.write(errors)
def process_locale_dir(self, locale_dir, files):
if self.domain == 'djangojs':
for file in files:
# We need to copy the JS files first, as otherwise babel will
# attempt to read package.json files in subdirs, such as
# base/package.json
in_path = urllib.parse.urljoin(
self.temp_dir_in + '/',
file.dirpath
)
os.makedirs(in_path, exist_ok=True)
in_file = urllib.parse.urljoin(
in_path + '/',
file.file
)
shutil.copy2(file.path, in_file)
out_path = urllib.parse.urljoin(
self.temp_dir_out + '/',
file.dirpath
)
file.dirpath = out_path
out, err, status = popen_wrapper([
'npm',
'run',
'babel-transform-template-literals',
'--',
'--out-dir',
self.temp_dir_out,
self.temp_dir_in
])
super().process_locale_dir(locale_dir, files)
def select_translatable_strings(files_list):
"""Returns translatable strings from the taken files list.
:param files_list: list of the paths to the files with
translatable strings
:return: PO file string with selected translatable strings
"""
# copy pasted with minimal changes from:
# line: 457, file: django.core.management.commands.makemessages
# for now we translate just Python files
args = [
'xgettext',
'-d', 'translatable_messages',
'--language=Python',
'--keyword=gettext_noop',
'--keyword=gettext_lazy',
'--keyword=ngettext_lazy:1,2',
'--keyword=ugettext_noop',
'--keyword=ugettext_lazy',
'--keyword=ungettext_lazy:1,2',
'--keyword=pgettext:1c,2',
'--keyword=npgettext:1c,2,3',
'--keyword=pgettext_lazy:1c,2',
'--keyword=npgettext_lazy:1c,2,3',
'--output=-',
]
with NamedTemporaryFile(mode='w+') as input_files_list:
input_files_list.write(('\n'.join(files_list)))
input_files_list.flush()
args.extend(['--files-from', input_files_list.name])
msgs, errors, status = popen_wrapper(args)
return msgs, errors
def gettext_version(self):
out, err, status = popen_wrapper(['xgettext', '--version'])
m = re.search(r'(\d)\.(\d+)\.(\d+)', out)
if m:
return tuple(int(d) for d in m.groups())
else:
raise CommandError("Unable to get gettext version. Is it installed?")
def gettext_version(self):
out, err, status = popen_wrapper(['xgettext', '--version'])
m = re.search(r'(\d)\.(\d+)\.?(\d+)?', out)
if m:
return tuple(int(d) for d in m.groups() if d is not None)
else:
raise CommandError("Unable to get gettext version. Is it installed?")
def compile_message_file(path):
"""Compiles a .po file into a .mo file by path."""
program = 'msgfmt'
if find_command(program) is None:
raise TranslationError(
"Can't find %s. Make sure you have GNU gettext tools 0.15 or newer installed."
% program)
def _has_bom(fn):
with open(fn, 'rb') as f:
sample = f.read(4)
return sample[:3] == b'\xef\xbb\xbf' or \
sample.startswith(codecs.BOM_UTF16_LE) or \
sample.startswith(codecs.BOM_UTF16_BE)
if _has_bom(path):
raise TranslationError(
"The %s file has a BOM (Byte Order Mark). Django only supports .po files encoded in UTF-8 and without any BOM."
% path)
pf = os.path.splitext(path)[0]
args = [
program, '--check-format', '-o',
npath(pf + '.mo'),
npath(pf + '.po')
]
output, errors, status = popen_wrapper(args)
if status:
if errors:
msg = "Execution of %s failed: %s" % (program, errors)
else:
msg = "Execution of %s failed" % program
raise TranslationError(msg)
def build_potfiles(self):
"""
Build pot files and apply msguniq to them.
"""
file_list = self.find_files(".")
self.remove_potfiles()
self.process_files(file_list)
potfiles = []
for path in self.locale_paths:
potfile = os.path.join(path, '%s.pot' % str(self.domain))
if not os.path.exists(potfile):
continue
args = ['msguniq'] + self.msguniq_options + [potfile]
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError(
"errors happened while running msguniq\n%s" % errors)
elif self.verbosity > 0:
self.stdout.write(errors)
msgs = normalize_eols(msgs)
with io.open(potfile, 'w', encoding='utf-8') as fp:
fp.write(msgs)
potfiles.append(potfile)
return potfiles
def build_potfiles(self):
"""
Build pot files and apply msguniq to them.
"""
file_list = self.find_files(".")
self.remove_potfiles()
for f in file_list:
try:
f.process(self, self.domain, self.extra_keywords)
except UnicodeDecodeError:
self.stdout.write("UnicodeDecodeError: skipped file %s in %s" % (f.file, f.dirpath))
potfiles = []
for path in self.locale_paths:
potfile = os.path.join(path, '%s.pot' % str(self.domain))
if not os.path.exists(potfile):
continue
args = ['msguniq', '--to-code=utf-8']
if self.wrap:
args.append(self.wrap)
if self.location:
args.append(self.location)
args.append(potfile)
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError(
"errors happened while running msguniq\n%s" % errors)
elif self.verbosity > 0:
self.stdout.write(errors)
with open(potfile, 'w') as fp:
fp.write(msgs)
potfiles.append(potfile)
return potfiles
def build_potfiles(self):
"""
Build pot files and apply msguniq to them.
"""
file_list = self.find_files(".")
self.remove_potfiles()
for f in file_list:
try:
f.process(self, self.domain)
except UnicodeDecodeError:
self.stdout.write("UnicodeDecodeError: skipped file %s in %s" %
(f.file, f.dirpath))
potfiles = []
for path in self.locale_paths:
potfile = os.path.join(path, '%s.pot' % str(self.domain))
if not os.path.exists(potfile):
continue
args = ['msguniq'] + self.msguniq_options + [potfile]
msgs, errors, status = popen_wrapper(args)
if six.PY2:
msgs = msgs.decode('utf-8')
if errors:
if status != STATUS_OK:
raise CommandError(
"errors happened while running msguniq\n%s" % errors)
elif self.verbosity > 0:
self.stdout.write(errors)
with io.open(potfile, 'w', encoding='utf-8') as fp:
fp.write(msgs)
potfiles.append(potfile)
return potfiles
def process_locale_dir(self, locale_dir, files):
if self.domain == "djangojs":
for file in files:
# We need to copy the JS files first, as otherwise babel will
# attempt to read package.json files in subdirs, such as
# base/package.json
in_path = urllib.parse.urljoin(self.temp_dir_in + "/",
file.dirpath)
os.makedirs(in_path, exist_ok=True)
in_file = urllib.parse.urljoin(in_path + "/", file.file)
shutil.copy2(file.path, in_file)
out_path = urllib.parse.urljoin(self.temp_dir_out + "/",
file.dirpath)
file.dirpath = out_path
os.chdir(".transpile/")
out, err, status = popen_wrapper([
"npm",
"run",
"babel-transform-template-literals",
"--",
"--out-dir",
self.temp_dir_out,
self.temp_dir_in,
])
os.chdir("../")
super().process_locale_dir(locale_dir, files)
def _compile(stdout, locale, basedir, program):
if locale:
dirs = [os.path.join(basedir, l, 'LC_MESSAGES') for l in locale]
else:
dirs = [basedir]
for ldir in dirs:
for dirpath, dirnames, filenames in os.walk(ldir):
for f in filenames:
if not f.endswith('.po'):
continue
stdout.write('processing file %s in %s\n' % (f, dirpath))
fn = os.path.join(dirpath, f)
if has_bom(fn):
raise CommandError(
"The %s file has a BOM (Byte Order Mark). Django only supports .po files encoded in UTF-8 and without any BOM."
% fn)
pf = os.path.splitext(fn)[0]
args = [
program, '--check-format', '-o',
npath(pf + '.mo'),
npath(pf + '.po')
]
output, errors, status = popen_wrapper(args)
if status:
if errors:
msg = "Execution of %s failed: %s" % (program, errors)
else:
msg = "Execution of %s failed" % program
raise CommandError(msg)
def compile_messages(self, locations):
"""
Locations is a list of tuples: [(directory, file), ...]
"""
for i, (dirpath, f) in enumerate(locations):
if self.verbosity > 0:
self.stdout.write('processing file %s in %s\n' % (f, dirpath))
po_path = os.path.join(dirpath, f)
if has_bom(po_path):
raise CommandError("The %s file has a BOM (Byte Order Mark). "
"Django only supports .po files encoded in "
"UTF-8 and without any BOM." % po_path)
base_path = os.path.splitext(po_path)[0]
# Check writability on first location
if i == 0 and not is_writable(npath(base_path + '.mo')):
self.stderr.write(
"The po files under %s are in a seemingly not writable location. "
"mo files will not be updated/created." % dirpath)
return
args = [self.program] + self.program_options + [
'-o', npath(base_path + '.mo'),
npath(base_path + '.po')
]
output, errors, status = popen_wrapper(args)
if status:
if errors:
msg = "Execution of %s failed: %s" % (self.program, errors)
else:
msg = "Execution of %s failed" % self.program
raise CommandError(msg)
def build_potfiles(self):
"""
Build pot files and apply msguniq to them.
"""
file_list = self.find_files(".")
self.remove_potfiles()
self.process_files(file_list)
potfiles = []
for path in self.locale_paths:
potfile = os.path.join(path, "%s.pot" % self.domain)
if not os.path.exists(potfile):
continue
args = ["msguniq"] + self.msguniq_options + [potfile]
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError(
"errors happened while running msguniq\n%s" % errors)
elif self.verbosity > 0:
self.stdout.write(errors)
msgs = normalize_eols(msgs)
with open(potfile, "w", encoding="utf-8") as fp:
fp.write(msgs)
potfiles.append(potfile)
return potfiles
def compile_messages(self, locations):
"""
Locations is a list of tuples: [(directory, file), ...]
"""
for i, (dirpath, f) in enumerate(locations):
if self.verbosity > 0:
self.stdout.write('processing file %s in %s\n' % (f, dirpath))
po_path = os.path.join(dirpath, f)
if has_bom(po_path):
raise CommandError("The %s file has a BOM (Byte Order Mark). "
"Django only supports .po files encoded in "
"UTF-8 and without any BOM." % po_path)
base_path = os.path.splitext(po_path)[0]
# Check writability on first location
if i == 0 and not is_writable(npath(base_path + '.mo')):
self.stderr.write("The po files under %s are in a seemingly not writable location. "
"mo files will not be updated/created." % dirpath)
return
args = [self.program] + self.program_options + ['-o',
npath(base_path + '.mo'), npath(base_path + '.po')]
output, errors, status = popen_wrapper(args)
if status:
if errors:
msg = "Execution of %s failed: %s" % (self.program, errors)
else:
msg = "Execution of %s failed" % self.program
raise CommandError(msg)
def write_po_file(self, potfile, locale):
"""
Creates or updates the PO file for self.domain and :param locale:.
Uses contents of the existing :param potfile:.
Uses msgmerge, and msgattrib GNU gettext utilities.
"""
basedir = os.path.join(os.path.dirname(potfile), locale, 'LC_MESSAGES')
if not os.path.isdir(basedir):
os.makedirs(basedir)
pofile = os.path.join(basedir, '%s.po' % str(self.domain))
if os.path.exists(pofile):
args = ['msgmerge'] + self.msgmerge_options + [pofile, potfile]
msgs, errors, status = popen_wrapper(args)
if six.PY2:
msgs = msgs.decode('utf-8')
if errors:
if status != STATUS_OK:
raise CommandError(
"errors happened while running msgmerge\n%s" % errors)
elif self.verbosity > 0:
self.stdout.write(errors)
else:
with io.open(potfile, 'r', encoding='utf-8') as fp:
msgs = fp.read()
if not self.invoked_for_django:
msgs = self.copy_plural_forms(msgs, locale)
msgs = msgs.replace(
"#. #-#-#-#-# %s.pot (PACKAGE VERSION) #-#-#-#-#\n" %
self.domain, "")
with io.open(pofile, 'w', encoding='utf-8') as fp:
fp.write(msgs)
if self.no_obsolete:
args = ['msgattrib'
] + self.msgattrib_options + ['-o', pofile, pofile]
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError(
"errors happened while running msgattrib\n%s" % errors)
elif self.verbosity > 0:
self.stdout.write(errors)
def handle(self, *args, **options):
use_underscores = True
try:
# Locale path and necessary settings
locale_path = settings.LOCALE_PATHS[0] if hasattr(settings,"LOCALE_PATHS") and isinstance(settings.LOCALE_PATHS,(list,tuple)) else settings.LOCALE_PATHS if hasattr(settings,"LOCALE_PATHS") else None #os.path.join(settings.BASE_DIR,"locale")
if not locale_path:
raise OneSkyApiClientException("LOCALE_PATHS not configured properly. Set your path to locale dir in settings.py as string")
if not hasattr(settings,"ONESKY_API_KEY") or not hasattr(settings,"ONESKY_API_SECRET"):
raise OneSkyApiClientException("ONESKY_API_KEY or ONESKY_API_SECRET not configured properly. Please include your OneSky key and secret in settings.py as string")
if not hasattr(settings,"ONESKY_PROJECTS") or not isinstance(settings.ONESKY_PROJECTS,list):
raise OneSkyApiClientException("ONESKY_PROJECTS not configured properly. Use list of OneSky project ids.")
# Init API client
client = OneSkyApiClient(api_key=settings.ONESKY_API_KEY, api_secret=settings.ONESKY_API_SECRET, locale_path=locale_path)
for locale_path, project_id in settings.ONESKY_PROJECTS:
client.locale_path = locale_path
# Get files
file_names = []
page = 1
while page:
status, json_response = client.file_list(project_id,page=page)
if status != 200:
raise OneSkyApiClientException("Unable to retrieve file list for #%s. OneSky API status: %s, OneSky API message: %s" % (project_id, status, json_response.get("meta",{}).get("message","")))
page = json_response.get("meta",{}).get("next_page",None)
file_names.extend([file.get("file_name") for file in json_response.get("data",[]) if file.get("file_name").endswith(".po")])
for file_name in file_names:
if isinstance(settings.LANGUAGES,(list,tuple)):
# language_codes = [language_item[0] for language_item in settings.LANGUAGES]
language_codes = [settings.LANGUAGE_CODE] #just upload the source language
if use_underscores:
language_codes = [l.replace('-','_') for l in language_codes]
for language_code in language_codes:
# Push each local file
upload_file_name = os.path.join(locale_path,language_code,"LC_MESSAGES",file_name)
if os.path.isfile(upload_file_name):
args = ['msgattrib', '--clear-fuzzy', '--empty', '-o', upload_file_name, upload_file_name]
msgs, errors, status = popen_wrapper(args)
# Upload to OneSky
if upload_file_name.endswith(".po"):
print "Uploading file: %s" % upload_file_name
client.file_upload(project_id, upload_file_name, file_format = "GNU_PO", locale = language_code, is_keeping_all_strings=False) # TODO: pass is_keeping_all_strings in command cli call
except OneSkyApiClientException,e:
print e
def write_po_file(self, potfile, locale):
"""
Create or update the PO file for self.domain and `locale`.
Use contents of the existing `potfile`.
Use msgmerge and msgattrib GNU gettext utilities.
"""
basedir = os.path.join(os.path.dirname(potfile), locale, "LC_MESSAGES")
os.makedirs(basedir, exist_ok=True)
pofile = os.path.join(basedir, "%s.po" % self.domain)
if os.path.exists(pofile):
args = ["msgmerge"] + self.msgmerge_options + [pofile, potfile]
_, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError(
"errors happened while running msgmerge\n%s" % errors)
elif self.verbosity > 0:
self.stdout.write(errors)
msgs = Path(pofile).read_text(encoding="utf-8")
else:
with open(potfile, encoding="utf-8") as fp:
msgs = fp.read()
if not self.invoked_for_django:
msgs = self.copy_plural_forms(msgs, locale)
msgs = normalize_eols(msgs)
msgs = msgs.replace(
"#. #-#-#-#-# %s.pot (PACKAGE VERSION) #-#-#-#-#\n" %
self.domain, "")
with open(pofile, "w", encoding="utf-8") as fp:
fp.write(msgs)
if self.no_obsolete:
args = ["msgattrib"
] + self.msgattrib_options + ["-o", pofile, pofile]
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError(
"errors happened while running msgattrib\n%s" % errors)
elif self.verbosity > 0:
self.stdout.write(errors)
def _run_cmd(self, cmd):
msgs, errors, status = popen_wrapper(shlex.split(cmd))
if errors:
if status != 0:
raise CommandError("Command `%s` error:\n%s" % (cmd, errors))
self.stderr.write(errors)
if msgs:
self.stdout.write(msgs)
if status != 0:
raise CommandError("Command `%s` exited with status %s" % (cmd, status))
def gettext_version(self):
# Gettext tools will output system-encoded bytestrings instead of UTF-8,
# when looking up the version. It's especially a problem on Windows.
out, err, status = popen_wrapper(
["xgettext", "--version"], stdout_encoding=DEFAULT_LOCALE_ENCODING)
m = re.search(r"(\d+)\.(\d+)\.?(\d+)?", out)
if m:
return tuple(int(d) for d in m.groups() if d is not None)
else:
raise CommandError(
"Unable to get gettext version. Is it installed?")
def _run_cmd(self, cmd):
msgs, errors, status = popen_wrapper(shlex.split(cmd))
if errors:
if status != 0:
raise CommandError("Command `%s` error:\n%s" % (cmd, errors))
self.stderr.write(errors)
if msgs:
self.stdout.write(msgs)
if status != 0:
raise CommandError("Command `%s` exited with status %s" %
(cmd, status))
def gettext_version(self):
# Gettext tools will output system-encoded bytestrings instead of UTF-8,
# when looking up the version. It's especially a problem on Windows.
out, err, status = popen_wrapper(
['xgettext', '--version'],
stdout_encoding=DEFAULT_LOCALE_ENCODING,
)
m = re.search(r'(\d+)\.(\d+)\.?(\d+)?', out)
if m:
return tuple(int(d) for d in m.groups() if d is not None)
else:
raise CommandError("Unable to get gettext version. Is it installed?")
def compile_messages(stdout, locale=None):
program = 'msgfmt'
if find_command(program) is None:
raise CommandError(
"Can't find %s. Make sure you have GNU gettext tools 0.15 or newer installed."
% program)
basedirs = [os.path.join('conf', 'locale'), 'locale']
if os.environ.get('DJANGO_SETTINGS_MODULE'):
from django.conf import settings
basedirs.extend(settings.LOCALE_PATHS)
# Gather existing directories.
basedirs = set(map(os.path.abspath, filter(os.path.isdir, basedirs)))
if not basedirs:
raise CommandError(
"This script should be run from the Django Git checkout or your project or app tree, or with the settings module specified."
)
for basedir in basedirs:
if locale:
dirs = [
os.path.join(basedir, l, 'LC_MESSAGES')
for l in (locale if isinstance(locale, list) else [locale])
]
else:
dirs = [basedir]
for ldir in dirs:
for dirpath, dirnames, filenames in os.walk(ldir):
for f in filenames:
if not f.endswith('.po'):
continue
stdout.write('processing file %s in %s\n' % (f, dirpath))
fn = os.path.join(dirpath, f)
if has_bom(fn):
raise CommandError(
"The %s file has a BOM (Byte Order Mark). Django only supports .po files encoded in UTF-8 and without any BOM."
% fn)
pf = os.path.splitext(fn)[0]
args = [
program, '--check-format', '-o',
npath(pf + '.mo'),
npath(pf + '.po')
]
output, errors, status = popen_wrapper(args)
if status:
if errors:
msg = "Execution of %s failed: %s" % (program,
errors)
else:
msg = "Execution of %s failed" % program
raise CommandError(msg)
def handle(self, *args, **options):
po_files = []
for file in listdir(self.french_locale_dir):
if file.endswith(".po"):
if file != "django.po":
po_files.append(path.join(self.french_locale_dir, file))
args = [self.program] + self.program_options + po_files + self.output_options
rc = popen_wrapper(args)
if rc[2] != 0:
print(rc[1])
else:
print("Created djangp.po")
def gettext_popen_wrapper(args, os_err_exc_type=CommandError):
"""
Makes sure text obtained from stdout of gettext utilities is Unicode.
"""
stdout, stderr, status_code = popen_wrapper(args, os_err_exc_type=os_err_exc_type)
if os.name == 'nt' and six.PY3:
# This looks weird because it's undoing what subprocess.Popen(universal_newlines=True).communicate()
# does when capturing PO files contents from stdout of gettext command line programs. See ticket #23271
# for details. No need to do anything on Python 2 because it's already a UTF-8-encoded byte-string there
stdout = stdout.encode(locale.getpreferredencoding(False)).decode('utf-8')
if six.PY2:
stdout = stdout.decode('utf-8')
return stdout, stderr, status_code
def gettext_popen_wrapper(args, os_err_exc_type=CommandError):
"""
Makes sure text obtained from stdout of gettext utilities is Unicode.
"""
stdout, stderr, status_code = popen_wrapper(args, os_err_exc_type=os_err_exc_type)
if os.name == 'nt' and six.PY3:
# This looks weird because it's undoing what subprocess.Popen(universal_newlines=True).communicate()
# does when capturing PO files contents from stdout of gettext command line programs. See ticket #23271
# for details. No need to do anything on Python 2 because it's already a UTF-8-encoded byte-string there
stdout = stdout.encode(locale.getpreferredencoding(False)).decode('utf-8')
if six.PY2:
stdout = stdout.decode('utf-8')
return stdout, stderr, status_code
def gettext_popen_wrapper(args, os_err_exc_type=CommandError, stdout_encoding="utf-8"):
"""
Makes sure text obtained from stdout of gettext utilities is Unicode.
"""
stdout, stderr, status_code = popen_wrapper(args, os_err_exc_type=os_err_exc_type)
if os.name == 'nt' and six.PY3 and stdout_encoding != DEFAULT_LOCALE_ENCODING:
# This looks weird because it's undoing what
# subprocess.Popen(universal_newlines=True).communicate()
# does when capturing PO files contents from stdout of gettext command
# line programs. No need to do anything on Python 2 because it's
# already a byte-string there (#23271).
stdout = stdout.encode(DEFAULT_LOCALE_ENCODING).decode(stdout_encoding)
if six.PY2:
stdout = stdout.decode(stdout_encoding)
return stdout, stderr, status_code
def gettext_popen_wrapper(args, os_err_exc_type=CommandError, stdout_encoding="utf-8"):
"""
Makes sure text obtained from stdout of gettext utilities is Unicode.
"""
stdout, stderr, status_code = popen_wrapper(args, os_err_exc_type=os_err_exc_type)
preferred_encoding = locale.getpreferredencoding(False)
if os.name == 'nt' and six.PY3 and stdout_encoding != preferred_encoding:
# This looks weird because it's undoing what
# subprocess.Popen(universal_newlines=True).communicate()
# does when capturing PO files contents from stdout of gettext command
# line programs. No need to do anything on Python 2 because it's
# already a byte-string there (#23271).
stdout = stdout.encode(preferred_encoding).decode(stdout_encoding)
if six.PY2:
stdout = stdout.decode(stdout_encoding)
return stdout, stderr, status_code
def handle(self, *args, **options):
custom_dir = path.join(self.BASE_DIR, 'search', 'templates',
'snippets', 'custom', options['search'])
output_path = path.join(self.BASE_DIR, 'locale', 'fr', 'LC_MESSAGES',
options['search'] + '.po')
temp_dir = TemporaryDirectory(dir=path.join(
self.BASE_DIR, 'search', 'templates', 'snippets', 'custom'))
template_files = []
for dirpath, dirnames, filenames in walk(custom_dir,
topdown=True,
onerror=None,
followlinks=False):
for filename in filenames:
file_path = path.normpath(path.join(dirpath, filename))
file_ext = path.splitext(filename)[1]
if file_ext in ['.html', '.py', '.txt']:
with open(file_path, encoding='utf-8') as fp:
src_data = fp.read()
rc = templatize(src_data, origin=dirpath)
temp_file = path.join(temp_dir.name, filename)
with open(temp_file, 'w', encoding='utf-8') as ofp:
ofp.write(rc)
template_files.append(temp_file)
args = [
'xgettext', '-d', options['search'], '--language=Python',
'--no-wrap', '--from-code=utf-8', '--sort-by-file',
'--copyright-holder="Government of Canada, Gouvernement du Canada"',
'--package-name="Open Canada Search - {0} Module"'.format(
options['search']), '--keyword=gettext_noop',
'--keyword=gettext_lazy', '--keyword=ngettext_lazy:1,2',
'--keyword=ugettext_noop', '--keyword=ugettext_lazy',
'--keyword=ungettext_lazy:1,2', '--keyword=pgettext:1c,2',
'--keyword=npgettext:1c,2,3', '--keyword=pgettext_lazy:1c,2',
'--keyword=npgettext_lazy:1c,2,3', '-o', output_path
]
#
if path.exists(output_path):
args += ['--join-existing']
args += template_files
rc = popen_wrapper(args)
if rc[2] != 0:
print(rc[1])
else:
print("Created {0}.po".format(options['search']))
temp_dir.cleanup()
def gettext_popen_wrapper(args, os_err_exc_type=CommandError, stdout_encoding="utf-8"):
"""
Makes sure text obtained from stdout of gettext utilities is Unicode.
"""
# This both decodes utf-8 and cleans line endings. Simply using
# popen_wrapper(universal_newlines=True) doesn't properly handle the
# encoding. This goes back to popen's flaky support for encoding:
# https://bugs.python.org/issue6135. This is a solution for #23271, #21928.
# No need to do anything on Python 2 because it's already a byte-string there.
manual_io_wrapper = six.PY3 and stdout_encoding != DEFAULT_LOCALE_ENCODING
stdout, stderr, status_code = popen_wrapper(args, os_err_exc_type=os_err_exc_type,
universal_newlines=not manual_io_wrapper)
if manual_io_wrapper:
stdout = io.TextIOWrapper(io.BytesIO(stdout), encoding=stdout_encoding).read()
if six.PY2:
stdout = stdout.decode(stdout_encoding)
return stdout, stderr, status_code
def gettext_popen_wrapper(args, os_err_exc_type=CommandError, stdout_encoding="utf-8"):
"""
Makes sure text obtained from stdout of gettext utilities is Unicode.
"""
# This both decodes utf-8 and cleans line endings. Simply using
# popen_wrapper(universal_newlines=True) doesn't properly handle the
# encoding. This goes back to popen's flaky support for encoding:
# https://bugs.python.org/issue6135. This is a solution for #23271, #21928.
# No need to do anything on Python 2 because it's already a byte-string there.
manual_io_wrapper = six.PY3 and stdout_encoding != DEFAULT_LOCALE_ENCODING
stdout, stderr, status_code = popen_wrapper(args, os_err_exc_type=os_err_exc_type,
universal_newlines=not manual_io_wrapper)
if manual_io_wrapper:
stdout = io.TextIOWrapper(io.BytesIO(stdout), encoding=stdout_encoding).read()
if six.PY2:
stdout = stdout.decode(stdout_encoding)
return stdout, stderr, status_code
def compile_messages(stdout, locale=None):
program = 'msgfmt'
if find_command(program) is None:
raise CommandError("Can't find %s. Make sure you have GNU gettext tools 0.15 or newer installed." % program)
basedirs = [os.path.join('conf', 'locale'), 'locale']
if os.environ.get('DJANGO_SETTINGS_MODULE'):
from django.conf import settings
basedirs.extend(settings.LOCALE_PATHS)
# Gather existing directories.
basedirs = set(map(os.path.abspath, filter(os.path.isdir, basedirs)))
if not basedirs:
raise CommandError("This script should be run from the Django Git checkout or your project or app tree, or with the settings module specified.")
for basedir in basedirs:
if locale:
dirs = [os.path.join(basedir, l, 'LC_MESSAGES') for l in locale]
else:
dirs = [basedir]
for ldir in dirs:
for dirpath, dirnames, filenames in os.walk(ldir):
for f in filenames:
if not f.endswith('.po'):
continue
stdout.write('processing file %s in %s\n' % (f, dirpath))
fn = os.path.join(dirpath, f)
if has_bom(fn):
raise CommandError("The %s file has a BOM (Byte Order Mark). Django only supports .po files encoded in UTF-8 and without any BOM." % fn)
pf = os.path.splitext(fn)[0]
args = [program, '--check-format', '-o', npath(pf + '.mo'), npath(pf + '.po')]
output, errors, status = popen_wrapper(args)
if status:
if errors:
msg = "Execution of %s failed: %s" % (program, errors)
else:
msg = "Execution of %s failed" % program
raise CommandError(msg)
def process_locale_dir(self, locale_dir, files):
if self.domain == 'djangojs':
for file in files:
# We need to copy the JS files first, as otherwise babel will
# attempt to read package.json files in subdirs, such as
# base/package.json
in_path = urllib.parse.urljoin(self.temp_dir_in + '/',
file.dirpath)
os.makedirs(in_path, exist_ok=True)
in_file = urllib.parse.urljoin(in_path + '/', file.file)
shutil.copy2(file.path, in_file)
out_path = urllib.parse.urljoin(self.temp_dir_out + '/',
file.dirpath)
file.dirpath = out_path
os.chdir('.transpile/')
out, err, status = popen_wrapper([
'npm', 'run', 'babel-transform-template-literals', '--',
'--out-dir', self.temp_dir_out, self.temp_dir_in
])
os.chdir('../')
super().process_locale_dir(locale_dir, files)
def handle(self, *args, **options):
BASE_STATIC = os.path.join(
settings.BASE_DIR, 'base', 'static', 'base'
)
program = 'lessc'
if find_command(program) is None:
raise CommandError(
'You need to install the LESS compiler (lessc) to use this '
'command.'
)
args = [
program,
os.path.join(BASE_STATIC, 'less', 'import.less'),
os.path.join(BASE_STATIC, 'css', 'style.css'),
]
output, errors, status = popen_wrapper(args)
if status:
if errors:
msg = '{program} execution failed: {errors}'
else:
msg = '{program} execution failed without error output.'
msg = msg.format(program=program, errors=errors)
raise CommandError(msg)
def compile_message_file(path):
"""Compiles a .po file into a .mo file by path."""
program = 'msgfmt'
if find_command(program) is None:
raise TranslationError("Can't find %s. Make sure you have GNU gettext tools 0.15 or newer installed." % program)
def _has_bom(fn):
with open(fn, 'rb') as f:
sample = f.read(4)
return sample[:3] == b'\xef\xbb\xbf' or \
sample.startswith(codecs.BOM_UTF16_LE) or \
sample.startswith(codecs.BOM_UTF16_BE)
if _has_bom(path):
raise TranslationError("The %s file has a BOM (Byte Order Mark). Django only supports .po files encoded in UTF-8 and without any BOM." % path)
pf = os.path.splitext(path)[0]
args = [program, '--check-format', '-o', npath(pf + '.mo'), npath(pf + '.po')]
output, errors, status = popen_wrapper(args)
if status:
if errors:
msg = "Execution of %s failed: %s" % (program, errors)
else:
msg = "Execution of %s failed" % program
raise TranslationError(msg)
def compile_messages(self, locations):
"""
Locations is a list of tuples: [(directory, file), ...]
"""
for i, (dirpath, f) in enumerate(locations):
if self.verbosity > 0:
self.stdout.write('processing file %s in %s\n' % (f, dirpath))
po_path = os.path.join(dirpath, f)
if has_bom(po_path):
self.stderr.write(
'The %s file has a BOM (Byte Order Mark). Django only '
'supports .po files encoded in UTF-8 and without any BOM.' % po_path
)
self.has_errors = True
continue
base_path = os.path.splitext(po_path)[0]
# Check writability on first location
if i == 0 and not is_writable(base_path + '.mo'):
self.stderr.write(
'The po files under %s are in a seemingly not writable location. '
'mo files will not be updated/created.' % dirpath
)
self.has_errors = True
return
args = [self.program] + self.program_options + [
'-o', base_path + '.mo', base_path + '.po'
]
output, errors, status = popen_wrapper(args)
if status:
if errors:
self.stderr.write('Execution of %s failed: %s.' % (self.program, errors))
else:
self.stderr.write('Execution of %s failed.' % self.program)
self.has_errors = True
def compile_messages(self, locations):
"""
Locations is a list of tuples: [(directory, file), ...]
"""
for i, (dirpath, f) in enumerate(locations):
if self.verbosity > 0:
self.stdout.write('processing file %s in %s\n' % (f, dirpath))
po_path = os.path.join(dirpath, f)
if has_bom(po_path):
self.stderr.write(
'The %s file has a BOM (Byte Order Mark). Django only '
'supports .po files encoded in UTF-8 and without any BOM.'
% po_path)
self.has_errors = True
continue
base_path = os.path.splitext(po_path)[0]
# Check writability on first location
if i == 0 and not is_writable(base_path + '.mo'):
self.stderr.write(
'The po files under %s are in a seemingly not writable location. '
'mo files will not be updated/created.' % dirpath)
self.has_errors = True
return
args = [self.program] + self.program_options + [
'-o', base_path + '.mo', base_path + '.po'
]
output, errors, status = popen_wrapper(args)
if status:
if errors:
self.stderr.write('Execution of %s failed: %s.' %
(self.program, errors))
else:
self.stderr.write('Execution of %s failed.' % self.program)
self.has_errors = True
from __future__ import unicode_literals
def process_locale_dir(self, locale_dir, files):
"""
Extract translatable literals from the specified files, creating or
updating the POT file for a given locale directory.
Use the xgettext GNU gettext utility.
"""
build_files = []
for translatable in files:
if self.verbosity > 1:
self.stdout.write("processing file %s in %s" %
(translatable.file, translatable.dirpath))
if self.domain not in ("djangojs", "django"):
continue
build_file = self.build_file_class(self, self.domain, translatable)
try:
build_file.preprocess()
except UnicodeDecodeError as e:
self.stdout.write(
"UnicodeDecodeError: skipped file %s in %s (reason: %s)" %
(
translatable.file,
translatable.dirpath,
e,
))
continue
except BaseException:
# Cleanup before exit.
for build_file in build_files:
build_file.cleanup()
raise
build_files.append(build_file)
if self.domain == "djangojs":
is_templatized = build_file.is_templatized
args = [
"xgettext",
"-d",
self.domain,
"--language=%s" % ("C" if is_templatized else "JavaScript", ),
"--keyword=gettext_noop",
"--keyword=gettext_lazy",
"--keyword=ngettext_lazy:1,2",
"--keyword=pgettext:1c,2",
"--keyword=npgettext:1c,2,3",
"--output=-",
]
elif self.domain == "django":
args = [
"xgettext",
"-d",
self.domain,
"--language=Python",
"--keyword=gettext_noop",
"--keyword=gettext_lazy",
"--keyword=ngettext_lazy:1,2",
"--keyword=pgettext:1c,2",
"--keyword=npgettext:1c,2,3",
"--keyword=pgettext_lazy:1c,2",
"--keyword=npgettext_lazy:1c,2,3",
"--output=-",
]
else:
return
input_files = [bf.work_path for bf in build_files]
with NamedTemporaryFile(mode="w+") as input_files_list:
input_files_list.write("\n".join(input_files))
input_files_list.flush()
args.extend(["--files-from", input_files_list.name])
args.extend(self.xgettext_options)
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
for build_file in build_files:
build_file.cleanup()
raise CommandError(
"errors happened while running xgettext on %s\n%s" %
("\n".join(input_files), errors))
elif self.verbosity > 0:
# Print warnings
self.stdout.write(errors)
if msgs:
if locale_dir is NO_LOCALE_DIR:
for build_file in build_files:
build_file.cleanup()
file_path = os.path.normpath(build_files[0].path)
raise CommandError(
"Unable to find a locale path to store translations for "
"file %s. Make sure the 'locale' directory exists in an "
"app or LOCALE_PATHS setting is set." % file_path)
for build_file in build_files:
msgs = build_file.postprocess_messages(msgs)
potfile = os.path.join(locale_dir, "%s.pot" % self.domain)
write_pot_file(potfile, msgs)
for build_file in build_files:
build_file.cleanup()
def process(self, command, potfile, domain, keep_pot=False):
"""
Extract translatable literals from self.file for :param domain:
creating or updating the :param potfile: POT file.
Uses the xgettext GNU gettext utility.
"""
from django.utils.translation import templatize
if command.verbosity > 1:
command.stdout.write("processing file %s in %s\n" % (self.file, self.dirpath))
_, file_ext = os.path.splitext(self.file)
if domain == "djangojs" and file_ext in command.extensions:
is_templatized = True
orig_file = os.path.join(self.dirpath, self.file)
with open(orig_file) as fp:
src_data = fp.read()
src_data = prepare_js_for_gettext(src_data)
thefile = "%s.c" % self.file
work_file = os.path.join(self.dirpath, thefile)
with open(work_file, "w") as fp:
fp.write(src_data)
args = [
"xgettext",
"-d",
domain,
"--language=C",
"--keyword=gettext_noop",
"--keyword=gettext_lazy",
"--keyword=ngettext_lazy:1,2",
"--keyword=pgettext:1c,2",
"--keyword=npgettext:1c,2,3",
"--from-code=UTF-8",
"--add-comments=Translators",
"--output=-",
]
if command.wrap:
args.append(command.wrap)
if command.location:
args.append(command.location)
args.append(work_file)
elif domain == "django" and (file_ext == ".py" or file_ext in command.extensions):
thefile = self.file
orig_file = os.path.join(self.dirpath, self.file)
is_templatized = file_ext in command.extensions
if is_templatized:
with open(orig_file, "rU") as fp:
src_data = fp.read()
thefile = "%s.py" % self.file
content = templatize(src_data, orig_file[2:])
with open(os.path.join(self.dirpath, thefile), "w") as fp:
fp.write(content)
work_file = os.path.join(self.dirpath, thefile)
args = [
"xgettext",
"-d",
domain,
"--language=Python",
"--keyword=gettext_noop",
"--keyword=gettext_lazy",
"--keyword=ngettext_lazy:1,2",
"--keyword=ugettext_noop",
"--keyword=ugettext_lazy",
"--keyword=ungettext_lazy:1,2",
"--keyword=pgettext:1c,2",
"--keyword=npgettext:1c,2,3",
"--keyword=pgettext_lazy:1c,2",
"--keyword=npgettext_lazy:1c,2,3",
"--from-code=UTF-8",
"--add-comments=Translators",
"--output=-",
]
if command.wrap:
args.append(command.wrap)
if command.location:
args.append(command.location)
args.append(work_file)
else:
return
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
if is_templatized:
os.unlink(work_file)
if not keep_pot and os.path.exists(potfile):
os.unlink(potfile)
raise CommandError("errors happened while running xgettext on %s\n%s" % (self.file, errors))
elif command.verbosity > 0:
# Print warnings
command.stdout.write(errors)
if msgs:
if is_templatized:
old = "#: " + work_file[2:]
new = "#: " + orig_file[2:]
msgs = msgs.replace(old, new)
write_pot_file(potfile, msgs)
if is_templatized:
os.unlink(work_file)
def test_no_existent_external_program(self):
msg = 'Error executing a_42_command_that_doesnt_exist_42'
with self.assertRaisesMessage(CommandError, msg):
popen_wrapper(['a_42_command_that_doesnt_exist_42'])
def test_no_existent_external_program(self):
with self.assertRaises(CommandError):
popen_wrapper(['a_42_command_that_doesnt_exist_42'])
def process(self, command, domain):
"""
Extract translatable literals from self.file for :param domain:,
creating or updating the POT file.
Uses the xgettext GNU gettext utility.
"""
from django.conf import settings
from django.utils.translation import templatize
if command.verbosity > 1:
command.stdout.write('processing file %s in %s\n' %
(self.file, self.dirpath))
_, file_ext = os.path.splitext(self.file)
if domain == 'djangojs' and file_ext in command.extensions:
is_templatized = True
orig_file = os.path.join(self.dirpath, self.file)
with io.open(orig_file, encoding=settings.FILE_CHARSET) as fp:
src_data = fp.read()
src_data = prepare_js_for_gettext(src_data)
thefile = '%s.c' % self.file
work_file = os.path.join(self.dirpath, thefile)
with io.open(work_file, "w", encoding='utf-8') as fp:
fp.write(src_data)
args = [
'xgettext', '-d', domain, '--language=C',
'--keyword=gettext_noop', '--keyword=gettext_lazy',
'--keyword=ngettext_lazy:1,2', '--keyword=pgettext:1c,2',
'--keyword=npgettext:1c,2,3', '--output=-'
] + command.xgettext_options
args.append(work_file)
elif domain == 'django' and (file_ext == '.py'
or file_ext in command.extensions):
thefile = self.file
orig_file = os.path.join(self.dirpath, self.file)
is_templatized = file_ext in command.extensions
if is_templatized:
with io.open(orig_file, 'r',
encoding=settings.FILE_CHARSET) as fp:
src_data = fp.read()
thefile = '%s.py' % self.file
content = templatize(src_data, orig_file[2:])
with io.open(os.path.join(self.dirpath, thefile),
"w",
encoding='utf-8') as fp:
fp.write(content)
work_file = os.path.join(self.dirpath, thefile)
args = [
'xgettext', '-d', domain, '--language=Python',
'--keyword=gettext_noop', '--keyword=gettext_lazy',
'--keyword=ngettext_lazy:1,2', '--keyword=ugettext_noop',
'--keyword=ugettext_lazy', '--keyword=ungettext_lazy:1,2',
'--keyword=pgettext:1c,2', '--keyword=npgettext:1c,2,3',
'--keyword=pgettext_lazy:1c,2',
'--keyword=npgettext_lazy:1c,2,3', '--output=-'
] + command.xgettext_options
args.append(work_file)
else:
return
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
if is_templatized:
os.unlink(work_file)
raise CommandError(
"errors happened while running xgettext on %s\n%s" %
(self.file, errors))
elif command.verbosity > 0:
# Print warnings
command.stdout.write(errors)
if msgs:
if six.PY2:
msgs = msgs.decode('utf-8')
# Write/append messages to pot file
potfile = os.path.join(self.locale_dir, '%s.pot' % str(domain))
if is_templatized:
# Remove '.py' suffix
if os.name == 'nt':
# Preserve '.\' prefix on Windows to respect gettext behavior
old = '#: ' + work_file
new = '#: ' + orig_file
else:
old = '#: ' + work_file[2:]
new = '#: ' + orig_file[2:]
msgs = msgs.replace(old, new)
write_pot_file(potfile, msgs)
if is_templatized:
os.unlink(work_file)
def handle_noargs(self, *args, **options):
locale = options.get('locale')
self.domain = options.get('domain')
self.verbosity = int(options.get('verbosity'))
process_all = options.get('all')
extensions = options.get('extensions')
self.symlinks = options.get('symlinks')
ignore_patterns = options.get('ignore_patterns')
if options.get('use_default_ignore_patterns'):
ignore_patterns += ['CVS', '.*', '*~', '*.pyc']
self.ignore_patterns = list(set(ignore_patterns))
self.wrap = '--no-wrap' if options.get('no_wrap') else ''
self.location = '--no-location' if options.get('no_location') else ''
self.no_obsolete = options.get('no_obsolete')
self.keep_pot = options.get('keep_pot')
if self.domain not in ('django', 'djangojs'):
raise CommandError("currently makemessages only supports domains "
"'django' and 'djangojs'")
if self.domain == 'djangojs':
exts = extensions if extensions else ['js']
else:
exts = extensions if extensions else ['html', 'txt']
self.extensions = handle_extensions(exts)
if (locale is None and not process_all) or self.domain is None:
raise CommandError("Type '%s help %s' for usage information." %
(os.path.basename(sys.argv[0]), sys.argv[1]))
if self.verbosity > 1:
self.stdout.write('examining files with the extensions: %s\n' %
get_text_list(list(self.extensions), 'and'))
# Need to ensure that the i18n framework is enabled
from django.conf import settings
if settings.configured:
settings.USE_I18N = True
else:
settings.configure(USE_I18N=True)
self.invoked_for_django = False
if os.path.isdir(os.path.join('conf', 'locale')):
localedir = os.path.abspath(os.path.join('conf', 'locale'))
self.invoked_for_django = True
# Ignoring all contrib apps
self.ignore_patterns += ['contrib/*']
elif os.path.isdir('locale'):
localedir = os.path.abspath('locale')
else:
raise CommandError(
"This script should be run from the Django Git "
"tree or your project or app tree. If you did indeed run it "
"from the Git checkout or your project or application, "
"maybe you are just missing the conf/locale (in the django "
"tree) or locale (for project and application) directory? It "
"is not created automatically, you have to create it by hand "
"if you want to enable i18n for your project or application.")
check_programs('xgettext')
# We require gettext version 0.15 or newer.
output, errors, status = popen_wrapper(['xgettext', '--version'])
if status != STATUS_OK:
raise CommandError(
"Error running xgettext. Note that Django "
"internationalization requires GNU gettext 0.15 or newer.")
match = re.search(r'(?P<major>\d+)\.(?P<minor>\d+)', output)
if match:
xversion = (int(match.group('major')), int(match.group('minor')))
if xversion < (0, 15):
raise CommandError(
"Django internationalization requires GNU "
"gettext 0.15 or newer. You are using version %s, please "
"upgrade your gettext toolset." % match.group())
potfile = self.build_pot_file(localedir)
# Build po files for each selected locale
locales = []
if locale is not None:
locales += locale.split(',') if not isinstance(locale,
list) else locale
elif process_all:
locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % localedir))
locales = [os.path.basename(l) for l in locale_dirs]
if locales:
check_programs('msguniq', 'msgmerge', 'msgattrib')
try:
for locale in locales:
if self.verbosity > 0:
self.stdout.write("processing locale %s\n" % locale)
self.write_po_file(potfile, locale)
finally:
if not self.keep_pot and os.path.exists(potfile):
os.unlink(potfile)
def test_no_existent_external_program(self):
with self.assertRaises(CommandError):
popen_wrapper(['a_42_command_that_doesnt_exist_42'])
def process(self, command, domain):
"""
Extract translatable literals from self.file for :param domain:,
creating or updating the POT file.
Uses the xgettext GNU gettext utility.
"""
from django.conf import settings
from django.utils.translation import templatize
if command.verbosity > 1:
command.stdout.write('processing file %s in %s\n' % (self.file, self.dirpath))
_, file_ext = os.path.splitext(self.file)
if domain == 'djangojs' and file_ext in command.extensions:
is_templatized = True
orig_file = os.path.join(self.dirpath, self.file)
with io.open(orig_file, encoding=settings.FILE_CHARSET) as fp:
src_data = fp.read()
src_data = prepare_js_for_gettext(src_data)
thefile = '%s.c' % self.file
work_file = os.path.join(self.dirpath, thefile)
with io.open(work_file, "w", encoding='utf-8') as fp:
fp.write(src_data)
args = [
'xgettext',
'-d', domain,
'--language=C',
'--keyword=gettext_noop',
'--keyword=gettext_lazy',
'--keyword=ngettext_lazy:1,2',
'--keyword=pgettext:1c,2',
'--keyword=npgettext:1c,2,3',
'--output=-'
] + command.xgettext_options
args.append(work_file)
elif domain == 'django' and (file_ext == '.py' or file_ext in command.extensions):
thefile = self.file
orig_file = os.path.join(self.dirpath, self.file)
is_templatized = file_ext in command.extensions
if is_templatized:
with io.open(orig_file, 'r', encoding=settings.FILE_CHARSET) as fp:
src_data = fp.read()
thefile = '%s.py' % self.file
content = templatize(src_data, orig_file[2:])
with io.open(os.path.join(self.dirpath, thefile), "w", encoding='utf-8') as fp:
fp.write(content)
work_file = os.path.join(self.dirpath, thefile)
args = [
'xgettext',
'-d', domain,
'--language=Python',
'--keyword=gettext_noop',
'--keyword=gettext_lazy',
'--keyword=ngettext_lazy:1,2',
'--keyword=ugettext_noop',
'--keyword=ugettext_lazy',
'--keyword=ungettext_lazy:1,2',
'--keyword=pgettext:1c,2',
'--keyword=npgettext:1c,2,3',
'--keyword=pgettext_lazy:1c,2',
'--keyword=npgettext_lazy:1c,2,3',
'--output=-'
] + command.xgettext_options
args.append(work_file)
else:
return
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
if is_templatized:
os.unlink(work_file)
raise CommandError(
"errors happened while running xgettext on %s\n%s" %
(self.file, errors))
elif command.verbosity > 0:
# Print warnings
command.stdout.write(errors)
if msgs:
if six.PY2:
msgs = msgs.decode('utf-8')
# Write/append messages to pot file
potfile = os.path.join(self.locale_dir, '%s.pot' % str(domain))
if is_templatized:
# Remove '.py' suffix
if os.name == 'nt':
# Preserve '.\' prefix on Windows to respect gettext behavior
old = '#: ' + work_file
new = '#: ' + orig_file
else:
old = '#: ' + work_file[2:]
new = '#: ' + orig_file[2:]
msgs = msgs.replace(old, new)
write_pot_file(potfile, msgs)
if is_templatized:
os.unlink(work_file)
from __future__ import unicode_literals
def process(self, command, potfile, domain, keep_pot=False):
"""
Extract translatable literals from self.file for :param domain:
creating or updating the :param potfile: POT file.
Uses the xgettext GNU gettext utility.
"""
from django.utils.translation import templatize
if command.verbosity > 1:
command.stdout.write('processing file %s in %s\n' % (self.file, self.dirpath))
_, file_ext = os.path.splitext(self.file)
if domain == 'djangojs' and file_ext in command.extensions:
is_templatized = True
orig_file = os.path.join(self.dirpath, self.file)
with open(orig_file) as fp:
src_data = fp.read()
src_data = prepare_js_for_gettext(src_data)
thefile = '%s.c' % self.file
work_file = os.path.join(self.dirpath, thefile)
with open(work_file, "w") as fp:
fp.write(src_data)
args = [
'xgettext',
'-d', domain,
'--language=C',
'--keyword=gettext_noop',
'--keyword=gettext_lazy',
'--keyword=ngettext_lazy:1,2',
'--keyword=pgettext:1c,2',
'--keyword=npgettext:1c,2,3',
'--from-code=UTF-8',
'--add-comments=Translators',
'--output=-'
]
if command.wrap:
args.append(command.wrap)
if command.location:
args.append(command.location)
args.append(work_file)
elif domain == 'django' and (file_ext == '.py' or file_ext in command.extensions):
thefile = self.file
orig_file = os.path.join(self.dirpath, self.file)
is_templatized = file_ext in command.extensions
if is_templatized:
with open(orig_file, "rU") as fp:
src_data = fp.read()
thefile = '%s.py' % self.file
content = templatize(src_data, orig_file[2:])
with open(os.path.join(self.dirpath, thefile), "w") as fp:
fp.write(content)
work_file = os.path.join(self.dirpath, thefile)
args = [
'xgettext',
'-d', domain,
'--language=Python',
'--keyword=gettext_noop',
'--keyword=gettext_lazy',
'--keyword=ngettext_lazy:1,2',
'--keyword=ugettext_noop',
'--keyword=ugettext_lazy',
'--keyword=ungettext_lazy:1,2',
'--keyword=pgettext:1c,2',
'--keyword=npgettext:1c,2,3',
'--keyword=pgettext_lazy:1c,2',
'--keyword=npgettext_lazy:1c,2,3',
'--from-code=UTF-8',
'--add-comments=Translators',
'--output=-'
]
if command.wrap:
args.append(command.wrap)
if command.location:
args.append(command.location)
args.append(work_file)
else:
return
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
if is_templatized:
os.unlink(work_file)
if not keep_pot and os.path.exists(potfile):
os.unlink(potfile)
raise CommandError(
"errors happened while running xgettext on %s\n%s" %
(self.file, errors))
elif command.verbosity > 0:
# Print warnings
command.stdout.write(errors)
if msgs:
if is_templatized:
# Remove '.py' suffix
if os.name =='nt':
# Preserve '.\' prefix on Windows to respect gettext behavior
old = '#: ' + work_file
new = '#: ' + orig_file
else:
old = '#: ' + work_file[2:]
new = '#: ' + orig_file[2:]
msgs = msgs.replace(old, new)
write_pot_file(potfile, msgs)
if is_templatized:
os.unlink(work_file)
def test_no_existent_external_program(self):
msg = 'Error executing a_42_command_that_doesnt_exist_42'
with self.assertRaisesMessage(CommandError, msg):
popen_wrapper(['a_42_command_that_doesnt_exist_42'])
def process_locale_dir(self, locale_dir, files):
"""
Extract translatable literals from the specified files, creating or
updating the POT file for a given locale directory.
Uses the xgettext GNU gettext utility.
"""
build_files = []
for translatable in files:
if self.verbosity > 1:
self.stdout.write('processing file %s in %s\n' % (
translatable.file, translatable.dirpath
))
if self.domain not in ('djangojs', 'django'):
continue
build_file = self.build_file_class(self, self.domain, translatable)
try:
build_file.preprocess()
except UnicodeDecodeError as e:
self.stdout.write(
'UnicodeDecodeError: skipped file %s in %s (reason: %s)' % (
translatable.file, translatable.dirpath, e,
)
)
continue
build_files.append(build_file)
if self.domain == 'djangojs':
is_templatized = build_file.is_templatized
args = [
'xgettext',
'-d', self.domain,
'--language=%s' % ('C' if is_templatized else 'JavaScript',),
'--keyword=gettext_noop',
'--keyword=gettext_lazy',
'--keyword=ngettext_lazy:1,2',
'--keyword=pgettext:1c,2',
'--keyword=npgettext:1c,2,3',
'--output=-',
]
elif self.domain == 'django':
args = [
'xgettext',
'-d', self.domain,
'--language=Python',
'--keyword=gettext_noop',
'--keyword=gettext_lazy',
'--keyword=ngettext_lazy:1,2',
'--keyword=ugettext_noop',
'--keyword=ugettext_lazy',
'--keyword=ungettext_lazy:1,2',
'--keyword=pgettext:1c,2',
'--keyword=npgettext:1c,2,3',
'--keyword=pgettext_lazy:1c,2',
'--keyword=npgettext_lazy:1c,2,3',
'--output=-',
]
else:
return
input_files = [bf.work_path for bf in build_files]
with NamedTemporaryFile(mode='w+') as input_files_list:
input_files_list.write('\n'.join(input_files))
input_files_list.flush()
args.extend(['--files-from', input_files_list.name])
args.extend(self.xgettext_options)
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
for build_file in build_files:
build_file.cleanup()
raise CommandError(
'errors happened while running xgettext on %s\n%s' %
('\n'.join(input_files), errors)
)
elif self.verbosity > 0:
# Print warnings
self.stdout.write(errors)
if msgs:
if locale_dir is NO_LOCALE_DIR:
file_path = os.path.normpath(build_files[0].path)
raise CommandError(
'Unable to find a locale path to store translations for '
'file %s' % file_path
)
for build_file in build_files:
msgs = build_file.postprocess_messages(msgs)
potfile = os.path.join(locale_dir, '%s.pot' % str(self.domain))
write_pot_file(potfile, msgs)
for build_file in build_files:
build_file.cleanup()
def handle_noargs(self, *args, **options):
locale = options.get("locale")
self.domain = options.get("domain")
self.verbosity = int(options.get("verbosity"))
process_all = options.get("all")
extensions = options.get("extensions")
self.symlinks = options.get("symlinks")
ignore_patterns = options.get("ignore_patterns")
if options.get("use_default_ignore_patterns"):
ignore_patterns += ["CVS", ".*", "*~", "*.pyc"]
self.ignore_patterns = list(set(ignore_patterns))
self.wrap = "--no-wrap" if options.get("no_wrap") else ""
self.location = "--no-location" if options.get("no_location") else ""
self.no_obsolete = options.get("no_obsolete")
self.keep_pot = options.get("keep_pot")
if self.domain not in ("django", "djangojs"):
raise CommandError("currently makemessages only supports domains " "'django' and 'djangojs'")
if self.domain == "djangojs":
exts = extensions if extensions else ["js"]
else:
exts = extensions if extensions else ["html", "txt"]
self.extensions = handle_extensions(exts)
if (locale is None and not process_all) or self.domain is None:
raise CommandError(
"Type '%s help %s' for usage information." % (os.path.basename(sys.argv[0]), sys.argv[1])
)
if self.verbosity > 1:
self.stdout.write("examining files with the extensions: %s\n" % get_text_list(list(self.extensions), "and"))
# Need to ensure that the i18n framework is enabled
from django.conf import settings
if settings.configured:
settings.USE_I18N = True
else:
settings.configure(USE_I18N=True)
self.invoked_for_django = False
if os.path.isdir(os.path.join("conf", "locale")):
localedir = os.path.abspath(os.path.join("conf", "locale"))
self.invoked_for_django = True
# Ignoring all contrib apps
self.ignore_patterns += ["contrib/*"]
elif os.path.isdir("locale"):
localedir = os.path.abspath("locale")
else:
raise CommandError(
"This script should be run from the Django Git "
"tree or your project or app tree. If you did indeed run it "
"from the Git checkout or your project or application, "
"maybe you are just missing the conf/locale (in the django "
"tree) or locale (for project and application) directory? It "
"is not created automatically, you have to create it by hand "
"if you want to enable i18n for your project or application."
)
check_programs("xgettext")
# We require gettext version 0.15 or newer.
output, errors, status = popen_wrapper(["xgettext", "--version"])
if status != STATUS_OK:
raise CommandError(
"Error running xgettext. Note that Django " "internationalization requires GNU gettext 0.15 or newer."
)
match = re.search(r"(?P<major>\d+)\.(?P<minor>\d+)", output)
if match:
xversion = (int(match.group("major")), int(match.group("minor")))
if xversion < (0, 15):
raise CommandError(
"Django internationalization requires GNU "
"gettext 0.15 or newer. You are using version %s, please "
"upgrade your gettext toolset." % match.group()
)
potfile = self.build_pot_file(localedir)
# Build po files for each selected locale
locales = []
if locale is not None:
locales += locale.split(",") if not isinstance(locale, list) else locale
elif process_all:
locale_dirs = filter(os.path.isdir, glob.glob("%s/*" % localedir))
locales = [os.path.basename(l) for l in locale_dirs]
if locales:
check_programs("msguniq", "msgmerge", "msgattrib")
try:
for locale in locales:
if self.verbosity > 0:
self.stdout.write("processing locale %s\n" % locale)
self.write_po_file(potfile, locale)
finally:
if not self.keep_pot and os.path.exists(potfile):
os.unlink(potfile)
