def test_no_detection_for_non_transifex(mock_find_files, mock_read, mock_push_strings): """No strings should be detected if a format other than Transifex Native is used in Python files and templates. """ mock_find_files.return_value = [ # 2 files with valid extension but no translatable content TranslatableFile('dir4/dir5', 'empty.py', 'locdir1'), TranslatableFile('dir4/dir5', 'empty.txt', 'locdir1'), ] mock_read.side_effect = [ # empty.py - shouldn't detect any strings as non-transifex PYTHON_TEMPLATE.format( _import='from django.utils.translation import ugettext_lazy as _', call1='_', call2='_', string1=u'A Django string', string2=u'Another Django string', ), # empty.txt - shouldn't detect any strings as non-transifex ( u'{% load i18n %}\n' u'{% trans "A Django string %}\n' u'{% blocktrans %}Another Django string{% endblocktrans %}', ), ] command = get_transifex_command() call_command(command, 'push', domain='djangojs') # command.string_collection.strings is like: {<key>: <SourceString>} found = command.subcommands['push'].string_collection.strings.values() assert set(found) == set([])
def test_replace_save_string_review(mock_find_files, mock_read, mock_prompt_file, mock_prompt_string, mock_save_file, mock_prompt_to_start1): mock_find_files.return_value = [ TranslatableFile('dir1/dir2', '1.html', 'locdir1'), TranslatableFile('dir4/dir5', '1.txt', 'locdir1'), ] mock_read.side_effect = [ HTML_SAMPLE_1, # 1.html HTML_SAMPLE_2, # 1.txt ] command = Command() call_command(command, 'migrate', save_policy='replace', review_policy='string') assert isinstance(command.subcommands['migrate'].executor.save_policy, ReplaceSavePolicy) assert isinstance(command.subcommands['migrate'].executor.review_policy, StringReviewPolicy) assert mock_prompt_file.call_count == 0 assert mock_prompt_string.call_count == 13 # The path and content that reached the save object # should have the correct values assert mock_save_file.call_args[0][0] == 'dir1/dir2/1.html' migration_compile = mock_save_file.call_args[0][1] assert migration_compile() == HTML_COMPILED_1
def test_backup_save_string_review(mock_find_files, mock_read, mock_prompt_file, mock_prompt_string, mock_save_file, mock_prompt_to_start1): mock_find_files.return_value = [ TranslatableFile('dir1/dir2', '1.html', 'locdir1'), TranslatableFile('dir4/dir5', '1.txt', 'locdir1'), ] mock_read.side_effect = [ HTML_SAMPLE_1, # 1.html HTML_SAMPLE_2, # 1.txt ] command = Command() call_command(command, 'migrate', save_policy='backup', review_policy='string') assert isinstance(command.subcommands['migrate'].executor.save_policy, BackupSavePolicy) assert isinstance(command.subcommands['migrate'].executor.review_policy, StringReviewPolicy) assert mock_prompt_file.call_count == 0 assert mock_prompt_string.call_count == 13 # 11 migrated strings # The path and content that reached the save object # should have the correct values, one for the backup and one for # the new content assert mock_save_file.call_args_list[0][0][0] == 'dir1/dir2/1.html.bak' original_content_getter = mock_save_file.call_args_list[0][0][1] assert original_content_getter() == HTML_SAMPLE_1 assert mock_save_file.call_args_list[1][0][0] == 'dir1/dir2/1.html' migration_compile = mock_save_file.call_args_list[1][0][1] assert migration_compile() == HTML_COMPILED_1
def test_python_parsing_success(mock_find_files, mock_read, mock_push_strings, mock_push_status): mock_push_strings.return_value = 202, PUSH_CDS_JSON mock_push_status.return_value = 200, PUSH_CDS_STATUS_JSON mock_find_files.return_value = [ TranslatableFile('dir1', '1.py', 'locdir1'), TranslatableFile('dir1/dir2', '2.py', 'locdir1'), TranslatableFile('dir1/dir3', '3.py', 'locdir1'), ] mock_read.side_effect = PYTHON_FILES expected = SOURCE_STRINGS run_and_compare(expected)
def test_python_parsing_no_strings(mock_extract, mock_find_files): mock_extract.return_value = [] mock_find_files.return_value = [ TranslatableFile('dir1', '1.py', 'locdir1'), TranslatableFile('dir1/dir2', '2.py', 'locdir1'), TranslatableFile('dir1/dir3', '3.py', 'locdir1'), ] command = get_transifex_command() call_command(command, 'push') # command.string_collection.strings is like: {<key>: <SourceString>} found = command.subcommands['push'].string_collection.strings.values() assert set(found) == set([])
def test_template_parsing(mock_find_files, mock_read, mock_push_strings): mock_find_files.return_value = [ TranslatableFile('dir1/dir2', '1.html', 'locdir1'), TranslatableFile('dir4/dir5', '1.txt', 'locdir1'), ] mock_read.side_effect = [ # 1.html HTML_TEMPLATE.replace( '{content}', u'<p>{% t "<b>Strong</b> {a}" a="A" _context="c1,c2" ' u'_tags="t1,t2" _comment="comment1" _charlimit=22 %}</p>\n' u'<p>{% ut "παράδειγμα {b}" b="B" _context="c1,c2" ' u'_tags="t1,t2" _comment="comment2" _charlimit=33 %}</p>' ), # 1.txt HTML_TEMPLATE.replace( '{content}', u'{% t _context="c1,c2" _tags="t1,t2" _comment="co1" _charlimit=22 %}\n' u'This is a short string\n' u'{% endt %}\n' u'{% t _context="c1,c2" _tags="t1,t2" _comment="co2" _charlimit=33 %}\n' u'This is not a shorter string\n' u'{% endt %}' ), ] expected = [ # 1.html SourceString( u'<b>Strong</b> {a}', 'c1,c2', _tags='t1,t2', _comment="comment1", _charlimit=22, _occurrences=['r1/dir2/1.html:4'], ), SourceString( u'παράδειγμα {b}', 'c1,c2', _tags='t1,t2', _comment="comment2", _charlimit=33, _occurrences=['r1/dir2/1.html:5'], ), # 1.txt SourceString( u'\nThis is a short string\n', 'c1,c2', _tags='t1,t2', _comment="co1", _charlimit=22, _occurrences=['r4/dir5/1.txt:4'], ), SourceString( u'\nThis is not a shorter string\n', 'c1,c2', _tags='t1,t2', _comment="co2", _charlimit=33, _occurrences=['r4/dir5/1.txt:7'], ), ] run_and_compare(expected)
def test_template_parsing2(mock_find_files, mock_read, mock_push_strings): mock_find_files.return_value = [ TranslatableFile('dir1/dir2', '1.html', 'locdir1'), ] mock_read.side_effect = [ # 1.html HTML_TEMPLATE.replace( '{content}', u'<p>{% t "<b>Strong</b> {a}" a="A" _context="c1,c2" ' u'_tags="t1,t2" _comment="comment1" _charlimit=22 ' u'_key="first_key" %}</p>\n' u'<p>{% ut "παράδειγμα {b}" b="B" _context="c1,c2" ' u'_tags="t1,t2" _comment="comment2" _charlimit=33 ' u'_key="second_key" %}</p>' ), ] expected = [ # 1.html SourceString( u'<b>Strong</b> {a}', 'c1,c2', _tags='t1,t2', _comment='comment1', _charlimit=22, _occurrences=['r1/dir2/1.html:4'], _key='first_key', ), SourceString( u'παράδειγμα {b}', 'c1,c2', _tags='t1,t2', _comment="comment2", _charlimit=33, _occurrences=['r1/dir2/1.html:5'], _key='second_key', ), ] run_and_compare(expected)
def test_without_tags_only(mock_find_files, mock_read, mock_push_strings, mock_push_status): mock_push_strings.return_value = 202, PUSH_CDS_JSON mock_push_status.return_value = 200, PUSH_CDS_STATUS_JSON mock_find_files.return_value = [ TranslatableFile('dir1', '1.py', 'locdir1'), TranslatableFile('dir1/dir2', '2.py', 'locdir1'), TranslatableFile('dir1/dir3', '3.py', 'locdir1'), ] mock_read.side_effect = PYTHON_FILES expected = [ clone_string(SOURCE_STRINGS[0]), clone_string(SOURCE_STRINGS[2]), clone_string(SOURCE_STRINGS[4]), ] run_and_compare(expected, without_tags_only='t1,t2')
def test_dry_run_save_none_review(mock_find_files, mock_read, mock_prompt_to_start1): mock_find_files.return_value = [ TranslatableFile('dir1/dir2', '1.html', 'locdir1'), TranslatableFile('dir4/dir5', '1.txt', 'locdir1'), ] mock_read.side_effect = [ HTML_SAMPLE_1, # 1.html HTML_SAMPLE_2, # 1.txt, ] command = Command() call_command(command, 'migrate', save_policy='none', review_policy='none') assert isinstance(command.subcommands['migrate'].executor.save_policy, NoopSavePolicy) assert isinstance(command.subcommands['migrate'].executor.review_policy, NoopReviewPolicy)
def test_dry_run(mock_find_files, mock_read, mock_push_strings): mock_push_strings.return_value = 200, {'doesnt': 'matter'} mock_find_files.return_value = [ TranslatableFile('dir1', '1.py', 'locdir1'), ] mock_read.side_effect = PYTHON_FILES command = get_transifex_command() call_command(command, 'push', dry_run=1) assert not mock_push_strings.called
def test_new_file_save_file_review(mock_find_files, mock_read, mock_prompt_file, mock_prompt_string, mock_save_file, mock_prompt_to_start1): mock_find_files.return_value = [ TranslatableFile('dir1/dir2', '1.html', 'locdir1'), TranslatableFile('dir4/dir5', '1.txt', 'locdir1'), TranslatableFile('dir4/dir5', '1.py', 'locdir1'), ] mock_read.side_effect = [ HTML_SAMPLE_1, # 1.html HTML_SAMPLE_2, # 1.txt PYTHON_SAMPLE, # 1.py ] command = Command() call_command(command, 'migrate', save_policy='new', review_policy='file') assert isinstance(command.subcommands['migrate'].executor.save_policy, NewFileSavePolicy) assert isinstance(command.subcommands['migrate'].executor.review_policy, FileReviewPolicy) # The first FileMigration instance that reached the review object # should compile to the proper Native syntax (1.html) file_migration1 = mock_prompt_file.call_args_list[0][0][0] assert file_migration1.compile() == HTML_COMPILED_1 # The same for 1.py file_migration2 = mock_prompt_file.call_args_list[1][0][0] assert file_migration2.compile() == PYTHON_SAMPLE_MIGRATED # No string review should have taken place assert mock_prompt_string.call_count == 0 # The path and content that reached the save object # should have the correct values assert mock_save_file.call_args_list[0][0][0] == 'dir1/dir2/1__native.html' migration_compile = mock_save_file.call_args_list[0][0][1] assert migration_compile() == HTML_COMPILED_1 assert mock_save_file.call_args_list[1][0][0] == 'dir4/dir5/1__native.py' migration_compile = mock_save_file.call_args_list[1][0][1] assert migration_compile() == PYTHON_SAMPLE_MIGRATED
def test_python_parsing_raises_unicode_error(mock_read, mock_find_files): o = b'\x00\x00' mock_read.side_effect = UnicodeDecodeError( 'funnycodec', o, 1, 2, 'This is just a fake reason!') mock_find_files.return_value = [ TranslatableFile('dir1', '1.py', 'locdir1'), ] command = get_transifex_command() call_command(command, 'push') # command.string_collection.strings is like: {<key>: <SourceString>} found = command.subcommands['push'].string_collection.strings.values() assert set(found) == set([])
def handle(self, *args, **options): self.domain = options['domain'] self.verbosity = options['verbosity'] self.ignore_patterns = [] self.path = options['path'] self.files = set(options['files'] or []) exts = MIGRATE_EXTENSIONS self.extensions = handle_extensions(exts) self.stats = { 'processed_files': 0, 'migrations': [], 'saved': [], 'errors': [], } # Create a reusable migrator for templates code self.django_migration_builder = DjangoTagMigrationBuilder() self.gettext_migration_builder = GettextMigrationBuilder( methods=GettextMethods(**GETTEXT_FUNCTIONS), import_statement=T_IMPORT, ) # -- Text mode: simply transform the given text and exit text = options['text'] if text: migrate_text(text, self._migrate_text) return # -- File mode: read all files based on the given options and migrate # each of them self.executor = MigrationExecutor( options, file_migrator_func=self._migrate_file, ) # Show an intro message self.executor.show_intro() # If specific files are defined, use those if self.files: dirpath = os.getcwd() files = [ TranslatableFile(dirpath, filename) for filename in self.files ] # Else search the file path for supported files else: files = self._find_files(self.path, 'migrate') # Execute the migration self.executor.migrate_files(files)
def test_append_tags(mock_find_files, mock_read, mock_push_strings, mock_push_status): """Test the functionality of the --append_tags option. The new tags should be added to the existing ones of each string. """ mock_push_strings.return_value = 202, PUSH_CDS_JSON mock_push_status.return_value = 200, PUSH_CDS_STATUS_JSON mock_find_files.return_value = [ TranslatableFile('dir1', '1.py', 'locdir1'), ] mock_read.side_effect = PYTHON_FILES expected = [ clone_string(SOURCE_STRINGS[0], new_tags=['extra1', 'extra2']), clone_string(SOURCE_STRINGS[1], new_tags=[ 't1', 't2', 'extra1', 'extra2']), ] run_and_compare(expected, append_tags=u'extra1,extra2')
def _find_files(self, root, subcommand): """Get all files in the given root. :param basestring root: the root path to search in :return: a list of TranslatableFile objects :rtype: list """ # TODO: See if we can remove functionality about locale dir and # simplify def is_ignored(path, ignore_patterns): """Check if the given path should be ignored or not.""" filename = os.path.basename(path) def ignore(pattern): return (fnmatch.fnmatchcase(filename, pattern) or fnmatch.fnmatchcase(path, pattern)) return any(ignore(pattern) for pattern in ignore_patterns) ignore_patterns = [os.path.normcase(p) for p in self.ignore_patterns] dir_suffixes = {'%s*' % path_sep for path_sep in {'/', os.sep}} norm_patterns = [] for p in ignore_patterns: for dir_suffix in dir_suffixes: if p.endswith(dir_suffix): norm_patterns.append(p[:-len(dir_suffix)]) break else: norm_patterns.append(p) all_files = [] ignored_roots = [] if self.settings_available: ignored_roots = [ os.path.normpath(p) for p in (settings.MEDIA_ROOT, settings.STATIC_ROOT) if p ] follow_links = self.symlinks if subcommand == 'push' else False for dirpath, dirnames, filenames in os.walk(root, topdown=True, followlinks=follow_links): for dirname in dirnames[:]: if (is_ignored( os.path.normpath(os.path.join(dirpath, dirname)), norm_patterns) or os.path.join(os.path.abspath(dirpath), dirname) in ignored_roots): dirnames.remove(dirname) self.verbose('Ignoring directory %s' % dirname) elif dirname == 'locale': dirnames.remove(dirname) if subcommand == 'push': self.locale_paths.insert( 0, os.path.join(os.path.abspath(dirpath), dirname)) for filename in filenames: file_path = os.path.normpath(os.path.join(dirpath, filename)) file_ext = os.path.splitext(filename)[1] if (file_ext not in self.extensions or is_ignored(file_path, self.ignore_patterns)): self.verbose('Ignoring file %s in %s' % (filename, dirpath)) else: if subcommand == 'push': locale_dir = None for path in self.locale_paths: if os.path.abspath(dirpath).\ startswith(os.path.dirname(path)): locale_dir = path break if not locale_dir: locale_dir = self.default_locale_path if not locale_dir: locale_dir = NO_LOCALE_DIR all_files.append( TranslatableFile(dirpath, filename, locale_dir)) elif subcommand == 'migrate': all_files.append( TranslatableFile(dirpath.lstrip('./'), filename)) return sorted(all_files)
def test_python_parsing_push_exception(mock_find_files, mock_read, mock_push_strings): mock_push_strings.return_value = 500, "content_to_trigger_exception" mock_find_files.return_value = [ TranslatableFile('dir1', '1.py', 'locdir1'), TranslatableFile('dir1/dir2', '2.py', 'locdir1'), TranslatableFile('dir1/dir3', '3.py', 'locdir1'), ] mock_read.side_effect = [ # 1.py PYTHON_TEMPLATE.format( _import='import transifex.native', call1='native.translate', call2='native.translate', string1=u'Le canapé', string2=u'Les données', ), # 2.py PYTHON_TEMPLATE.format( _import='import transifex.native as _n', call1='_n.translate', call2='_n.translate', string1=u'Le canapé 2', string2=u'Les données 2', ), # 3.py PYTHON_TEMPLATE.format( _import='from transifex.native import translate', call1='translate', call2='translate', string1=u'Le canapé 3', string2=u'Les données 3', ), ] expected = [ # 1.py SourceString( u'Le canapé', u'désign1,désign2', _occurrences=['r1/1.py:6'], ), SourceString( u'Les données', u'opération', _comment='comment', _tags='t1,t2', _charlimit=33, _occurrences=['r1/1.py:7'], ), # 2.py SourceString( u'Le canapé 2', u'désign1,désign2', _occurrences=['r1/dir2/2.py:6'], ), SourceString( u'Les données 2', u'opération', _comment='comment', _tags='t1,t2', _charlimit=33, _occurrences=['r1/dir2/2.py:7'], ), # 3.py SourceString( u'Le canapé 3', u'désign1,désign2', _occurrences=['r1/dir3/3.py:6'], ), SourceString( u'Les données 3', u'opération', _comment='comment', _tags='t1,t2', _charlimit=33, _occurrences=['r1/dir3/3.py:7'], ), ] compare(expected)