def test_pot_charset_header_is_utf8(self):
"""Content-Type: ... charset=CHARSET is replaced with charset=UTF-8"""
msgs = ('# SOME DESCRIPTIVE TITLE.\n'
'# (some lines truncated as they are not relevant)\n'
'"Content-Type: text/plain; charset=CHARSET\\n"\n'
'"Content-Transfer-Encoding: 8bit\\n"\n'
'\n'
'#: somefile.py:8\n'
'msgid "mañana; charset=CHARSET"\n'
'msgstr ""\n')
with tempfile.NamedTemporaryFile() as pot_file:
pot_filename = pot_file.name
write_pot_file(pot_filename, msgs)
with open(pot_filename, encoding='utf-8') as fp:
pot_contents = fp.read()
self.assertIn('Content-Type: text/plain; charset=UTF-8',
pot_contents)
self.assertIn('mañana; charset=CHARSET', pot_contents)
def test_pot_charset_header_is_utf8(self):
"""Content-Type: ... charset=CHARSET is replaced with charset=UTF-8"""
msgs = (
'# SOME DESCRIPTIVE TITLE.\n'
'# (some lines truncated as they are not relevant)\n'
'"Content-Type: text/plain; charset=CHARSET\\n"\n'
'"Content-Transfer-Encoding: 8bit\\n"\n'
'\n'
'#: somefile.py:8\n'
'msgid "mañana; charset=CHARSET"\n'
'msgstr ""\n'
)
with tempfile.NamedTemporaryFile() as pot_file:
pot_filename = pot_file.name
write_pot_file(pot_filename, msgs)
with open(pot_filename, encoding='utf-8') as fp:
pot_contents = fp.read()
self.assertIn('Content-Type: text/plain; charset=UTF-8', pot_contents)
self.assertIn('mañana; charset=CHARSET', pot_contents)
def custom_process_locale_dir(self, locale_dir, files):
"""
Extract translatable literals from the specified files, creating or
updating the POT file for a given locale directory.
Uses the xgettext GNU gettext utility.
"""
build_files = []
for translatable in files:
if self.verbosity > 1:
self.stdout.write('processing file %s in %s\n' % (
translatable.file, translatable.dirpath
))
if self.domain != 'djangular':
continue
build_file = self.build_file_class(self, self.domain, translatable)
try:
build_file.preprocess()
except UnicodeDecodeError as e:
self.stdout.write(
'UnicodeDecodeError: skipped file %s in %s (reason: %s)' % (
translatable.file, translatable.dirpath, e,
)
)
continue
build_files.append(build_file)
if self.domain == 'djangular':
# self.domain = 'django'
args = [
'xgettext',
'-d', self.domain,
'--language=Python',
'--keyword=gettext_noop',
'--keyword=gettext_lazy',
'--keyword=ngettext_lazy:1,2',
'--keyword=ugettext_noop',
'--keyword=ugettext_lazy',
'--keyword=ungettext_lazy:1,2',
'--keyword=pgettext:1c,2',
'--keyword=npgettext:1c,2,3',
'--keyword=pgettext_lazy:1c,2',
'--keyword=npgettext_lazy:1c,2,3',
'--output=-',
]
else:
return
input_files = [bf.work_path for bf in build_files]
with NamedTemporaryFile(mode='w+') as input_files_list:
input_files_list.write('\n'.join(input_files))
input_files_list.flush()
args.extend(['--files-from', input_files_list.name])
args.extend(self.xgettext_options)
msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
for build_file in build_files:
build_file.cleanup()
raise CommandError(
'errors happened while running xgettext on %s\n%s' %
('\n'.join(input_files), errors)
)
elif self.verbosity > 0:
# Print warnings
self.stdout.write(errors)
if msgs:
if locale_dir is NO_LOCALE_DIR:
file_path = os.path.normpath(build_files[0].path)
raise CommandError(
'Unable to find a locale path to store translations for '
'file %s' % file_path
)
for build_file in build_files:
msgs = build_file.postprocess_messages(msgs)
potfile = os.path.join(locale_dir, '%s.pot' % str(self.domain))
write_pot_file(potfile, msgs)
self.domain = 'djangular'
for build_file in build_files:
build_file.cleanup()
def process_file(file, dirpath, potfile, domain, verbosity, #@ReservedAssignment
extensions, wrap, location, stdout=sys.stdout,
extra_keywords=None):
"""
Extract translatable literals from :param file: for :param domain:
creating or updating the :param potfile: POT file.
Uses the xgettext GNU gettext utility.
"""
from django.utils.translation import templatize
if verbosity > 1:
stdout.write('processing file %s in %s\n' % (file, dirpath))
_, file_ext = os.path.splitext(file)
if domain == 'djangojs' and file_ext in extensions:
is_templatized = True
orig_file = os.path.join(dirpath, file)
src_data = open(orig_file).read()
src_data = prepare_js_for_gettext(src_data)
thefile = '%s.c' % file
work_file = os.path.join(dirpath, thefile)
f = open(work_file, "w")
try:
f.write(src_data)
finally:
f.close()
keywords = ['gettext_noop',
'gettext_lazy',
'ngettext_lazy:1,2',
'pgettext:1c,2',
'npgettext:1c,2,3',
]
if extra_keywords:
keywords.extend(extra_keywords)
cmd = 'xgettext -d %s -L C %s %s'
for kw in keywords:
cmd += ' --keyword=%s' % kw
cmd += ' --from-code UTF-8 --add-comments=Translators -o - "%s"'
cmd %= (domain, wrap, location, work_file)
elif domain == 'django' and (file_ext == '.py' or file_ext in extensions):
thefile = file
orig_file = os.path.join(dirpath, file)
is_templatized = file_ext in extensions
if is_templatized:
src_data = open(orig_file, "rU").read()
thefile = '%s.py' % file
content = templatize(src_data, orig_file[2:])
f = open(os.path.join(dirpath, thefile), "w")
try:
f.write(content)
finally:
f.close()
work_file = os.path.join(dirpath, thefile)
keywords = ['gettext_lazy',
'ngettext_lazy:1,2',
'ugettext_noop',
'ugettext_lazy',
'ungettext_lazy:1,2',
'pgettext:1c,2',
'npgettext:1c,2,3',
'pgettext_lazy:1c,2',
'npgettext_lazy:1c,2,3',
]
if extra_keywords:
keywords.extend(extra_keywords)
cmd = 'xgettext -d %s -L Python %s %s'
for kw in keywords:
cmd += ' --keyword=%s' % kw
cmd += ' --from-code UTF-8 --add-comments=Translators -o - "%s"'
cmd %= (domain, wrap, location, work_file)
else:
return
msgs, errors = makemessages._popen(cmd)
if errors:
if is_templatized:
os.unlink(work_file)
if os.path.exists(potfile):
os.unlink(potfile)
raise CommandError(
"errors happened while running xgettext on %s\n%s" %
(file, errors))
if msgs:
makemessages.write_pot_file(potfile, msgs, orig_file, work_file, is_templatized)
if is_templatized:
os.unlink(work_file)
def process(self, command, domain):
"""
Override django TranslatableFile behavior for adding djangular domain
"""
file_ext = os.path.splitext(self.file)[1]
if domain == 'djangular' and file_ext != '.py':
orig_file = os.path.join(self.dirpath, self.file)
work_file = orig_file
with io.open(orig_file, encoding=settings.FILE_CHARSET) as fp:
src_data = fp.read()
content = djangularize(src_data, orig_file[2:])
work_file = os.path.join(self.dirpath, '%s.py' % self.file)
with io.open(work_file, "w", encoding='utf-8') as fp:
fp.write(unicode(content))
args = [
'xgettext',
'-d', domain,
'--language=Python',
'--keyword=gettext_noop',
'--keyword=gettext_lazy',
'--keyword=ngettext_lazy:1,2',
'--keyword=ugettext_noop',
'--keyword=ugettext_lazy',
'--keyword=ungettext_lazy:1,2',
'--keyword=pgettext:1c,2',
'--keyword=npgettext:1c,2,3',
'--keyword=pgettext_lazy:1c,2',
'--keyword=npgettext_lazy:1c,2,3',
'--output=-'
] + command.xgettext_options
args.append(work_file)
msgs, errors, status = makemessages.gettext_popen_wrapper(args)
if errors:
if status != makemessages.STATUS_OK:
os.unlink(work_file)
raise CommandError(
"errors happened while running xgettext on %s\n%s" %
(self.file, errors))
elif command.verbosity > 0:
# Print warnings
command.stdout.write(errors)
if msgs:
# Write/append messages to pot file
if self.locale_dir is NO_LOCALE_DIR:
file_path = os.path.normpath(os.path.join(self.dirpath,
self.file))
raise CommandError(
"Unable to find a locale path to store"
" translations for file %s" % file_path)
potfile = os.path.join(self.locale_dir, '%s.pot' % str(domain))
# Remove '.py' suffix
if os.name == 'nt':
# Preserve '.\' prefix on Windows to respect gettext behavior
old = '#: ' + work_file
new = '#: ' + orig_file
else:
old = '#: ' + work_file[2:]
new = '#: ' + orig_file[2:]
msgs = msgs.replace(old, new)
makemessages.write_pot_file(potfile, msgs)
os.unlink(work_file)
else:
super(DjangularTranslatableFile,
self).process(command, domain)
def process_file(
file,
dirpath,
potfile,
domain,
verbosity, #@ReservedAssignment
extensions,
wrap,
location,
stdout=sys.stdout,
extra_keywords=None):
"""
Extract translatable literals from :param file: for :param domain:
creating or updating the :param potfile: POT file.
Uses the xgettext GNU gettext utility.
"""
from django.utils.translation import templatize
if verbosity > 1:
stdout.write('processing file %s in %s\n' % (file, dirpath))
_, file_ext = os.path.splitext(file)
if domain == 'djangojs' and file_ext in extensions:
is_templatized = True
orig_file = os.path.join(dirpath, file)
src_data = open(orig_file).read()
src_data = prepare_js_for_gettext(src_data)
thefile = '%s.c' % file
work_file = os.path.join(dirpath, thefile)
f = open(work_file, "w")
try:
f.write(src_data)
finally:
f.close()
keywords = [
'gettext_noop',
'gettext_lazy',
'ngettext_lazy:1,2',
'pgettext:1c,2',
'npgettext:1c,2,3',
]
if extra_keywords:
keywords.extend(extra_keywords)
cmd = 'xgettext -d %s -L C %s %s'
for kw in keywords:
cmd += ' --keyword=%s' % kw
cmd += ' --from-code UTF-8 --add-comments=Translators -o - "%s"'
cmd %= (domain, wrap, location, work_file)
elif domain == 'django' and (file_ext == '.py' or file_ext in extensions):
thefile = file
orig_file = os.path.join(dirpath, file)
is_templatized = file_ext in extensions
if is_templatized:
src_data = open(orig_file, "rU").read()
thefile = '%s.py' % file
content = templatize(src_data, orig_file[2:])
f = open(os.path.join(dirpath, thefile), "w")
try:
f.write(content)
finally:
f.close()
work_file = os.path.join(dirpath, thefile)
keywords = [
'gettext_lazy',
'ngettext_lazy:1,2',
'ugettext_noop',
'ugettext_lazy',
'ungettext_lazy:1,2',
'pgettext:1c,2',
'npgettext:1c,2,3',
'pgettext_lazy:1c,2',
'npgettext_lazy:1c,2,3',
]
if extra_keywords:
keywords.extend(extra_keywords)
cmd = 'xgettext -d %s -L Python %s %s'
for kw in keywords:
cmd += ' --keyword=%s' % kw
cmd += ' --from-code UTF-8 --add-comments=Translators -o - "%s"'
cmd %= (domain, wrap, location, work_file)
else:
return
msgs, errors = makemessages._popen(cmd)
if errors:
if is_templatized:
os.unlink(work_file)
if os.path.exists(potfile):
os.unlink(potfile)
raise CommandError("errors happened while running xgettext on %s\n%s" %
(file, errors))
if msgs:
makemessages.write_pot_file(potfile, msgs, orig_file, work_file,
is_templatized)
if is_templatized:
os.unlink(work_file)