def test_pot_charset_header_is_utf8(self): """Content-Type: ... charset=CHARSET is replaced with charset=UTF-8""" msgs = ('# SOME DESCRIPTIVE TITLE.\n' '# (some lines truncated as they are not relevant)\n' '"Content-Type: text/plain; charset=CHARSET\\n"\n' '"Content-Transfer-Encoding: 8bit\\n"\n' '\n' '#: somefile.py:8\n' 'msgid "mañana; charset=CHARSET"\n' 'msgstr ""\n') with tempfile.NamedTemporaryFile() as pot_file: pot_filename = pot_file.name write_pot_file(pot_filename, msgs) with open(pot_filename, encoding='utf-8') as fp: pot_contents = fp.read() self.assertIn('Content-Type: text/plain; charset=UTF-8', pot_contents) self.assertIn('mañana; charset=CHARSET', pot_contents)
def test_pot_charset_header_is_utf8(self): """Content-Type: ... charset=CHARSET is replaced with charset=UTF-8""" msgs = ( '# SOME DESCRIPTIVE TITLE.\n' '# (some lines truncated as they are not relevant)\n' '"Content-Type: text/plain; charset=CHARSET\\n"\n' '"Content-Transfer-Encoding: 8bit\\n"\n' '\n' '#: somefile.py:8\n' 'msgid "mañana; charset=CHARSET"\n' 'msgstr ""\n' ) with tempfile.NamedTemporaryFile() as pot_file: pot_filename = pot_file.name write_pot_file(pot_filename, msgs) with open(pot_filename, encoding='utf-8') as fp: pot_contents = fp.read() self.assertIn('Content-Type: text/plain; charset=UTF-8', pot_contents) self.assertIn('mañana; charset=CHARSET', pot_contents)
def custom_process_locale_dir(self, locale_dir, files): """ Extract translatable literals from the specified files, creating or updating the POT file for a given locale directory. Uses the xgettext GNU gettext utility. """ build_files = [] for translatable in files: if self.verbosity > 1: self.stdout.write('processing file %s in %s\n' % ( translatable.file, translatable.dirpath )) if self.domain != 'djangular': continue build_file = self.build_file_class(self, self.domain, translatable) try: build_file.preprocess() except UnicodeDecodeError as e: self.stdout.write( 'UnicodeDecodeError: skipped file %s in %s (reason: %s)' % ( translatable.file, translatable.dirpath, e, ) ) continue build_files.append(build_file) if self.domain == 'djangular': # self.domain = 'django' args = [ 'xgettext', '-d', self.domain, '--language=Python', '--keyword=gettext_noop', '--keyword=gettext_lazy', '--keyword=ngettext_lazy:1,2', '--keyword=ugettext_noop', '--keyword=ugettext_lazy', '--keyword=ungettext_lazy:1,2', '--keyword=pgettext:1c,2', '--keyword=npgettext:1c,2,3', '--keyword=pgettext_lazy:1c,2', '--keyword=npgettext_lazy:1c,2,3', '--output=-', ] else: return input_files = [bf.work_path for bf in build_files] with NamedTemporaryFile(mode='w+') as input_files_list: input_files_list.write('\n'.join(input_files)) input_files_list.flush() args.extend(['--files-from', input_files_list.name]) args.extend(self.xgettext_options) msgs, errors, status = popen_wrapper(args) if errors: if status != STATUS_OK: for build_file in build_files: build_file.cleanup() raise CommandError( 'errors happened while running xgettext on %s\n%s' % ('\n'.join(input_files), errors) ) elif self.verbosity > 0: # Print warnings self.stdout.write(errors) if msgs: if locale_dir is NO_LOCALE_DIR: file_path = os.path.normpath(build_files[0].path) raise CommandError( 'Unable to find a locale path to store translations for ' 'file %s' % file_path ) for build_file in build_files: msgs = build_file.postprocess_messages(msgs) potfile = os.path.join(locale_dir, '%s.pot' % str(self.domain)) write_pot_file(potfile, msgs) self.domain = 'djangular' for build_file in build_files: build_file.cleanup()
def process_file(file, dirpath, potfile, domain, verbosity, #@ReservedAssignment extensions, wrap, location, stdout=sys.stdout, extra_keywords=None): """ Extract translatable literals from :param file: for :param domain: creating or updating the :param potfile: POT file. Uses the xgettext GNU gettext utility. """ from django.utils.translation import templatize if verbosity > 1: stdout.write('processing file %s in %s\n' % (file, dirpath)) _, file_ext = os.path.splitext(file) if domain == 'djangojs' and file_ext in extensions: is_templatized = True orig_file = os.path.join(dirpath, file) src_data = open(orig_file).read() src_data = prepare_js_for_gettext(src_data) thefile = '%s.c' % file work_file = os.path.join(dirpath, thefile) f = open(work_file, "w") try: f.write(src_data) finally: f.close() keywords = ['gettext_noop', 'gettext_lazy', 'ngettext_lazy:1,2', 'pgettext:1c,2', 'npgettext:1c,2,3', ] if extra_keywords: keywords.extend(extra_keywords) cmd = 'xgettext -d %s -L C %s %s' for kw in keywords: cmd += ' --keyword=%s' % kw cmd += ' --from-code UTF-8 --add-comments=Translators -o - "%s"' cmd %= (domain, wrap, location, work_file) elif domain == 'django' and (file_ext == '.py' or file_ext in extensions): thefile = file orig_file = os.path.join(dirpath, file) is_templatized = file_ext in extensions if is_templatized: src_data = open(orig_file, "rU").read() thefile = '%s.py' % file content = templatize(src_data, orig_file[2:]) f = open(os.path.join(dirpath, thefile), "w") try: f.write(content) finally: f.close() work_file = os.path.join(dirpath, thefile) keywords = ['gettext_lazy', 'ngettext_lazy:1,2', 'ugettext_noop', 'ugettext_lazy', 'ungettext_lazy:1,2', 'pgettext:1c,2', 'npgettext:1c,2,3', 'pgettext_lazy:1c,2', 'npgettext_lazy:1c,2,3', ] if extra_keywords: keywords.extend(extra_keywords) cmd = 'xgettext -d %s -L Python %s %s' for kw in keywords: cmd += ' --keyword=%s' % kw cmd += ' --from-code UTF-8 --add-comments=Translators -o - "%s"' cmd %= (domain, wrap, location, work_file) else: return msgs, errors = makemessages._popen(cmd) if errors: if is_templatized: os.unlink(work_file) if os.path.exists(potfile): os.unlink(potfile) raise CommandError( "errors happened while running xgettext on %s\n%s" % (file, errors)) if msgs: makemessages.write_pot_file(potfile, msgs, orig_file, work_file, is_templatized) if is_templatized: os.unlink(work_file)
def process(self, command, domain): """ Override django TranslatableFile behavior for adding djangular domain """ file_ext = os.path.splitext(self.file)[1] if domain == 'djangular' and file_ext != '.py': orig_file = os.path.join(self.dirpath, self.file) work_file = orig_file with io.open(orig_file, encoding=settings.FILE_CHARSET) as fp: src_data = fp.read() content = djangularize(src_data, orig_file[2:]) work_file = os.path.join(self.dirpath, '%s.py' % self.file) with io.open(work_file, "w", encoding='utf-8') as fp: fp.write(unicode(content)) args = [ 'xgettext', '-d', domain, '--language=Python', '--keyword=gettext_noop', '--keyword=gettext_lazy', '--keyword=ngettext_lazy:1,2', '--keyword=ugettext_noop', '--keyword=ugettext_lazy', '--keyword=ungettext_lazy:1,2', '--keyword=pgettext:1c,2', '--keyword=npgettext:1c,2,3', '--keyword=pgettext_lazy:1c,2', '--keyword=npgettext_lazy:1c,2,3', '--output=-' ] + command.xgettext_options args.append(work_file) msgs, errors, status = makemessages.gettext_popen_wrapper(args) if errors: if status != makemessages.STATUS_OK: os.unlink(work_file) raise CommandError( "errors happened while running xgettext on %s\n%s" % (self.file, errors)) elif command.verbosity > 0: # Print warnings command.stdout.write(errors) if msgs: # Write/append messages to pot file if self.locale_dir is NO_LOCALE_DIR: file_path = os.path.normpath(os.path.join(self.dirpath, self.file)) raise CommandError( "Unable to find a locale path to store" " translations for file %s" % file_path) potfile = os.path.join(self.locale_dir, '%s.pot' % str(domain)) # Remove '.py' suffix if os.name == 'nt': # Preserve '.\' prefix on Windows to respect gettext behavior old = '#: ' + work_file new = '#: ' + orig_file else: old = '#: ' + work_file[2:] new = '#: ' + orig_file[2:] msgs = msgs.replace(old, new) makemessages.write_pot_file(potfile, msgs) os.unlink(work_file) else: super(DjangularTranslatableFile, self).process(command, domain)
def process_file( file, dirpath, potfile, domain, verbosity, #@ReservedAssignment extensions, wrap, location, stdout=sys.stdout, extra_keywords=None): """ Extract translatable literals from :param file: for :param domain: creating or updating the :param potfile: POT file. Uses the xgettext GNU gettext utility. """ from django.utils.translation import templatize if verbosity > 1: stdout.write('processing file %s in %s\n' % (file, dirpath)) _, file_ext = os.path.splitext(file) if domain == 'djangojs' and file_ext in extensions: is_templatized = True orig_file = os.path.join(dirpath, file) src_data = open(orig_file).read() src_data = prepare_js_for_gettext(src_data) thefile = '%s.c' % file work_file = os.path.join(dirpath, thefile) f = open(work_file, "w") try: f.write(src_data) finally: f.close() keywords = [ 'gettext_noop', 'gettext_lazy', 'ngettext_lazy:1,2', 'pgettext:1c,2', 'npgettext:1c,2,3', ] if extra_keywords: keywords.extend(extra_keywords) cmd = 'xgettext -d %s -L C %s %s' for kw in keywords: cmd += ' --keyword=%s' % kw cmd += ' --from-code UTF-8 --add-comments=Translators -o - "%s"' cmd %= (domain, wrap, location, work_file) elif domain == 'django' and (file_ext == '.py' or file_ext in extensions): thefile = file orig_file = os.path.join(dirpath, file) is_templatized = file_ext in extensions if is_templatized: src_data = open(orig_file, "rU").read() thefile = '%s.py' % file content = templatize(src_data, orig_file[2:]) f = open(os.path.join(dirpath, thefile), "w") try: f.write(content) finally: f.close() work_file = os.path.join(dirpath, thefile) keywords = [ 'gettext_lazy', 'ngettext_lazy:1,2', 'ugettext_noop', 'ugettext_lazy', 'ungettext_lazy:1,2', 'pgettext:1c,2', 'npgettext:1c,2,3', 'pgettext_lazy:1c,2', 'npgettext_lazy:1c,2,3', ] if extra_keywords: keywords.extend(extra_keywords) cmd = 'xgettext -d %s -L Python %s %s' for kw in keywords: cmd += ' --keyword=%s' % kw cmd += ' --from-code UTF-8 --add-comments=Translators -o - "%s"' cmd %= (domain, wrap, location, work_file) else: return msgs, errors = makemessages._popen(cmd) if errors: if is_templatized: os.unlink(work_file) if os.path.exists(potfile): os.unlink(potfile) raise CommandError("errors happened while running xgettext on %s\n%s" % (file, errors)) if msgs: makemessages.write_pot_file(potfile, msgs, orig_file, work_file, is_templatized) if is_templatized: os.unlink(work_file)