def test_no_detection_for_non_transifex(mock_find_files, mock_read, mock_push_strings):
"""No strings should be detected if a format other than Transifex Native
is used in Python files and templates.
"""
mock_find_files.return_value = [
# 2 files with valid extension but no translatable content
TranslatableFile('dir4/dir5', 'empty.py', 'locdir1'),
TranslatableFile('dir4/dir5', 'empty.txt', 'locdir1'),
]
mock_read.side_effect = [
# empty.py - shouldn't detect any strings as non-transifex
PYTHON_TEMPLATE.format(
_import='from django.utils.translation import ugettext_lazy as _',
call1='_',
call2='_',
string1=u'A Django string',
string2=u'Another Django string',
),
# empty.txt - shouldn't detect any strings as non-transifex
(
u'{% load i18n %}\n'
u'{% trans "A Django string %}\n'
u'{% blocktrans %}Another Django string{% endblocktrans %}',
),
]
command = get_transifex_command()
call_command(command, 'push', domain='djangojs')
# command.string_collection.strings is like: {<key>: <SourceString>}
found = command.subcommands['push'].string_collection.strings.values()
assert set(found) == set([])
def test_replace_save_string_review(mock_find_files, mock_read,
mock_prompt_file, mock_prompt_string,
mock_save_file, mock_prompt_to_start1):
mock_find_files.return_value = [
TranslatableFile('dir1/dir2', '1.html', 'locdir1'),
TranslatableFile('dir4/dir5', '1.txt', 'locdir1'),
]
mock_read.side_effect = [
HTML_SAMPLE_1, # 1.html
HTML_SAMPLE_2, # 1.txt
]
command = Command()
call_command(command,
'migrate',
save_policy='replace',
review_policy='string')
assert isinstance(command.subcommands['migrate'].executor.save_policy,
ReplaceSavePolicy)
assert isinstance(command.subcommands['migrate'].executor.review_policy,
StringReviewPolicy)
assert mock_prompt_file.call_count == 0
assert mock_prompt_string.call_count == 13
# The path and content that reached the save object
# should have the correct values
assert mock_save_file.call_args[0][0] == 'dir1/dir2/1.html'
migration_compile = mock_save_file.call_args[0][1]
assert migration_compile() == HTML_COMPILED_1
def test_backup_save_string_review(mock_find_files, mock_read,
mock_prompt_file, mock_prompt_string,
mock_save_file, mock_prompt_to_start1):
mock_find_files.return_value = [
TranslatableFile('dir1/dir2', '1.html', 'locdir1'),
TranslatableFile('dir4/dir5', '1.txt', 'locdir1'),
]
mock_read.side_effect = [
HTML_SAMPLE_1, # 1.html
HTML_SAMPLE_2, # 1.txt
]
command = Command()
call_command(command,
'migrate',
save_policy='backup',
review_policy='string')
assert isinstance(command.subcommands['migrate'].executor.save_policy,
BackupSavePolicy)
assert isinstance(command.subcommands['migrate'].executor.review_policy,
StringReviewPolicy)
assert mock_prompt_file.call_count == 0
assert mock_prompt_string.call_count == 13 # 11 migrated strings
# The path and content that reached the save object
# should have the correct values, one for the backup and one for
# the new content
assert mock_save_file.call_args_list[0][0][0] == 'dir1/dir2/1.html.bak'
original_content_getter = mock_save_file.call_args_list[0][0][1]
assert original_content_getter() == HTML_SAMPLE_1
assert mock_save_file.call_args_list[1][0][0] == 'dir1/dir2/1.html'
migration_compile = mock_save_file.call_args_list[1][0][1]
assert migration_compile() == HTML_COMPILED_1
def test_python_parsing_success(mock_find_files, mock_read, mock_push_strings,
mock_push_status):
mock_push_strings.return_value = 202, PUSH_CDS_JSON
mock_push_status.return_value = 200, PUSH_CDS_STATUS_JSON
mock_find_files.return_value = [
TranslatableFile('dir1', '1.py', 'locdir1'),
TranslatableFile('dir1/dir2', '2.py', 'locdir1'),
TranslatableFile('dir1/dir3', '3.py', 'locdir1'),
]
mock_read.side_effect = PYTHON_FILES
expected = SOURCE_STRINGS
run_and_compare(expected)
def test_python_parsing_no_strings(mock_extract, mock_find_files):
mock_extract.return_value = []
mock_find_files.return_value = [
TranslatableFile('dir1', '1.py', 'locdir1'),
TranslatableFile('dir1/dir2', '2.py', 'locdir1'),
TranslatableFile('dir1/dir3', '3.py', 'locdir1'),
]
command = get_transifex_command()
call_command(command, 'push')
# command.string_collection.strings is like: {<key>: <SourceString>}
found = command.subcommands['push'].string_collection.strings.values()
assert set(found) == set([])
def test_template_parsing(mock_find_files, mock_read, mock_push_strings):
mock_find_files.return_value = [
TranslatableFile('dir1/dir2', '1.html', 'locdir1'),
TranslatableFile('dir4/dir5', '1.txt', 'locdir1'),
]
mock_read.side_effect = [
# 1.html
HTML_TEMPLATE.replace(
'{content}',
u'<p>{% t "<b>Strong</b> {a}" a="A" _context="c1,c2" '
u'_tags="t1,t2" _comment="comment1" _charlimit=22 %}</p>\n'
u'<p>{% ut "παράδειγμα {b}" b="B" _context="c1,c2" '
u'_tags="t1,t2" _comment="comment2" _charlimit=33 %}</p>'
),
# 1.txt
HTML_TEMPLATE.replace(
'{content}',
u'{% t _context="c1,c2" _tags="t1,t2" _comment="co1" _charlimit=22 %}\n'
u'This is a short string\n'
u'{% endt %}\n'
u'{% t _context="c1,c2" _tags="t1,t2" _comment="co2" _charlimit=33 %}\n'
u'This is not a shorter string\n'
u'{% endt %}'
),
]
expected = [
# 1.html
SourceString(
u'<b>Strong</b> {a}', 'c1,c2', _tags='t1,t2', _comment="comment1",
_charlimit=22, _occurrences=['r1/dir2/1.html:4'],
),
SourceString(
u'παράδειγμα {b}', 'c1,c2', _tags='t1,t2', _comment="comment2",
_charlimit=33, _occurrences=['r1/dir2/1.html:5'],
),
# 1.txt
SourceString(
u'\nThis is a short string\n', 'c1,c2', _tags='t1,t2',
_comment="co1", _charlimit=22, _occurrences=['r4/dir5/1.txt:4'],
),
SourceString(
u'\nThis is not a shorter string\n', 'c1,c2', _tags='t1,t2',
_comment="co2", _charlimit=33, _occurrences=['r4/dir5/1.txt:7'],
),
]
run_and_compare(expected)
def test_template_parsing2(mock_find_files, mock_read, mock_push_strings):
mock_find_files.return_value = [
TranslatableFile('dir1/dir2', '1.html', 'locdir1'),
]
mock_read.side_effect = [
# 1.html
HTML_TEMPLATE.replace(
'{content}',
u'<p>{% t "<b>Strong</b> {a}" a="A" _context="c1,c2" '
u'_tags="t1,t2" _comment="comment1" _charlimit=22 '
u'_key="first_key" %}</p>\n'
u'<p>{% ut "παράδειγμα {b}" b="B" _context="c1,c2" '
u'_tags="t1,t2" _comment="comment2" _charlimit=33 '
u'_key="second_key" %}</p>'
),
]
expected = [
# 1.html
SourceString(
u'<b>Strong</b> {a}', 'c1,c2', _tags='t1,t2', _comment='comment1',
_charlimit=22, _occurrences=['r1/dir2/1.html:4'],
_key='first_key',
),
SourceString(
u'παράδειγμα {b}', 'c1,c2', _tags='t1,t2', _comment="comment2",
_charlimit=33, _occurrences=['r1/dir2/1.html:5'],
_key='second_key',
),
]
run_and_compare(expected)
def test_without_tags_only(mock_find_files, mock_read, mock_push_strings, mock_push_status):
mock_push_strings.return_value = 202, PUSH_CDS_JSON
mock_push_status.return_value = 200, PUSH_CDS_STATUS_JSON
mock_find_files.return_value = [
TranslatableFile('dir1', '1.py', 'locdir1'),
TranslatableFile('dir1/dir2', '2.py', 'locdir1'),
TranslatableFile('dir1/dir3', '3.py', 'locdir1'),
]
mock_read.side_effect = PYTHON_FILES
expected = [
clone_string(SOURCE_STRINGS[0]),
clone_string(SOURCE_STRINGS[2]),
clone_string(SOURCE_STRINGS[4]),
]
run_and_compare(expected, without_tags_only='t1,t2')
def test_dry_run_save_none_review(mock_find_files, mock_read,
mock_prompt_to_start1):
mock_find_files.return_value = [
TranslatableFile('dir1/dir2', '1.html', 'locdir1'),
TranslatableFile('dir4/dir5', '1.txt', 'locdir1'),
]
mock_read.side_effect = [
HTML_SAMPLE_1, # 1.html
HTML_SAMPLE_2, # 1.txt,
]
command = Command()
call_command(command, 'migrate', save_policy='none', review_policy='none')
assert isinstance(command.subcommands['migrate'].executor.save_policy,
NoopSavePolicy)
assert isinstance(command.subcommands['migrate'].executor.review_policy,
NoopReviewPolicy)
def test_dry_run(mock_find_files, mock_read, mock_push_strings):
mock_push_strings.return_value = 200, {'doesnt': 'matter'}
mock_find_files.return_value = [
TranslatableFile('dir1', '1.py', 'locdir1'),
]
mock_read.side_effect = PYTHON_FILES
command = get_transifex_command()
call_command(command, 'push', dry_run=1)
assert not mock_push_strings.called
def test_new_file_save_file_review(mock_find_files, mock_read,
mock_prompt_file, mock_prompt_string,
mock_save_file, mock_prompt_to_start1):
mock_find_files.return_value = [
TranslatableFile('dir1/dir2', '1.html', 'locdir1'),
TranslatableFile('dir4/dir5', '1.txt', 'locdir1'),
TranslatableFile('dir4/dir5', '1.py', 'locdir1'),
]
mock_read.side_effect = [
HTML_SAMPLE_1, # 1.html
HTML_SAMPLE_2, # 1.txt
PYTHON_SAMPLE, # 1.py
]
command = Command()
call_command(command, 'migrate', save_policy='new', review_policy='file')
assert isinstance(command.subcommands['migrate'].executor.save_policy,
NewFileSavePolicy)
assert isinstance(command.subcommands['migrate'].executor.review_policy,
FileReviewPolicy)
# The first FileMigration instance that reached the review object
# should compile to the proper Native syntax (1.html)
file_migration1 = mock_prompt_file.call_args_list[0][0][0]
assert file_migration1.compile() == HTML_COMPILED_1
# The same for 1.py
file_migration2 = mock_prompt_file.call_args_list[1][0][0]
assert file_migration2.compile() == PYTHON_SAMPLE_MIGRATED
# No string review should have taken place
assert mock_prompt_string.call_count == 0
# The path and content that reached the save object
# should have the correct values
assert mock_save_file.call_args_list[0][0][0] == 'dir1/dir2/1__native.html'
migration_compile = mock_save_file.call_args_list[0][0][1]
assert migration_compile() == HTML_COMPILED_1
assert mock_save_file.call_args_list[1][0][0] == 'dir4/dir5/1__native.py'
migration_compile = mock_save_file.call_args_list[1][0][1]
assert migration_compile() == PYTHON_SAMPLE_MIGRATED
def test_python_parsing_raises_unicode_error(mock_read, mock_find_files):
o = b'\x00\x00'
mock_read.side_effect = UnicodeDecodeError(
'funnycodec', o, 1, 2, 'This is just a fake reason!')
mock_find_files.return_value = [
TranslatableFile('dir1', '1.py', 'locdir1'),
]
command = get_transifex_command()
call_command(command, 'push')
# command.string_collection.strings is like: {<key>: <SourceString>}
found = command.subcommands['push'].string_collection.strings.values()
assert set(found) == set([])
def handle(self, *args, **options):
self.domain = options['domain']
self.verbosity = options['verbosity']
self.ignore_patterns = []
self.path = options['path']
self.files = set(options['files'] or [])
exts = MIGRATE_EXTENSIONS
self.extensions = handle_extensions(exts)
self.stats = {
'processed_files': 0,
'migrations': [],
'saved': [],
'errors': [],
}
# Create a reusable migrator for templates code
self.django_migration_builder = DjangoTagMigrationBuilder()
self.gettext_migration_builder = GettextMigrationBuilder(
methods=GettextMethods(**GETTEXT_FUNCTIONS),
import_statement=T_IMPORT,
)
# -- Text mode: simply transform the given text and exit
text = options['text']
if text:
migrate_text(text, self._migrate_text)
return
# -- File mode: read all files based on the given options and migrate
# each of them
self.executor = MigrationExecutor(
options,
file_migrator_func=self._migrate_file,
)
# Show an intro message
self.executor.show_intro()
# If specific files are defined, use those
if self.files:
dirpath = os.getcwd()
files = [
TranslatableFile(dirpath, filename) for filename in self.files
]
# Else search the file path for supported files
else:
files = self._find_files(self.path, 'migrate')
# Execute the migration
self.executor.migrate_files(files)
def test_append_tags(mock_find_files, mock_read, mock_push_strings, mock_push_status):
"""Test the functionality of the --append_tags option.
The new tags should be added to the existing ones of each string.
"""
mock_push_strings.return_value = 202, PUSH_CDS_JSON
mock_push_status.return_value = 200, PUSH_CDS_STATUS_JSON
mock_find_files.return_value = [
TranslatableFile('dir1', '1.py', 'locdir1'),
]
mock_read.side_effect = PYTHON_FILES
expected = [
clone_string(SOURCE_STRINGS[0], new_tags=['extra1', 'extra2']),
clone_string(SOURCE_STRINGS[1], new_tags=[
't1', 't2', 'extra1', 'extra2']),
]
run_and_compare(expected, append_tags=u'extra1,extra2')
def _find_files(self, root, subcommand):
"""Get all files in the given root.
:param basestring root: the root path to search in
:return: a list of TranslatableFile objects
:rtype: list
"""
# TODO: See if we can remove functionality about locale dir and
# simplify
def is_ignored(path, ignore_patterns):
"""Check if the given path should be ignored or not."""
filename = os.path.basename(path)
def ignore(pattern):
return (fnmatch.fnmatchcase(filename, pattern)
or fnmatch.fnmatchcase(path, pattern))
return any(ignore(pattern) for pattern in ignore_patterns)
ignore_patterns = [os.path.normcase(p) for p in self.ignore_patterns]
dir_suffixes = {'%s*' % path_sep for path_sep in {'/', os.sep}}
norm_patterns = []
for p in ignore_patterns:
for dir_suffix in dir_suffixes:
if p.endswith(dir_suffix):
norm_patterns.append(p[:-len(dir_suffix)])
break
else:
norm_patterns.append(p)
all_files = []
ignored_roots = []
if self.settings_available:
ignored_roots = [
os.path.normpath(p)
for p in (settings.MEDIA_ROOT, settings.STATIC_ROOT) if p
]
follow_links = self.symlinks if subcommand == 'push' else False
for dirpath, dirnames, filenames in os.walk(root,
topdown=True,
followlinks=follow_links):
for dirname in dirnames[:]:
if (is_ignored(
os.path.normpath(os.path.join(dirpath, dirname)),
norm_patterns)
or os.path.join(os.path.abspath(dirpath),
dirname) in ignored_roots):
dirnames.remove(dirname)
self.verbose('Ignoring directory %s' % dirname)
elif dirname == 'locale':
dirnames.remove(dirname)
if subcommand == 'push':
self.locale_paths.insert(
0, os.path.join(os.path.abspath(dirpath), dirname))
for filename in filenames:
file_path = os.path.normpath(os.path.join(dirpath, filename))
file_ext = os.path.splitext(filename)[1]
if (file_ext not in self.extensions
or is_ignored(file_path, self.ignore_patterns)):
self.verbose('Ignoring file %s in %s' %
(filename, dirpath))
else:
if subcommand == 'push':
locale_dir = None
for path in self.locale_paths:
if os.path.abspath(dirpath).\
startswith(os.path.dirname(path)):
locale_dir = path
break
if not locale_dir:
locale_dir = self.default_locale_path
if not locale_dir:
locale_dir = NO_LOCALE_DIR
all_files.append(
TranslatableFile(dirpath, filename, locale_dir))
elif subcommand == 'migrate':
all_files.append(
TranslatableFile(dirpath.lstrip('./'), filename))
return sorted(all_files)
def test_python_parsing_push_exception(mock_find_files, mock_read,
mock_push_strings):
mock_push_strings.return_value = 500, "content_to_trigger_exception"
mock_find_files.return_value = [
TranslatableFile('dir1', '1.py', 'locdir1'),
TranslatableFile('dir1/dir2', '2.py', 'locdir1'),
TranslatableFile('dir1/dir3', '3.py', 'locdir1'),
]
mock_read.side_effect = [
# 1.py
PYTHON_TEMPLATE.format(
_import='import transifex.native',
call1='native.translate',
call2='native.translate',
string1=u'Le canapé',
string2=u'Les données',
),
# 2.py
PYTHON_TEMPLATE.format(
_import='import transifex.native as _n',
call1='_n.translate',
call2='_n.translate',
string1=u'Le canapé 2',
string2=u'Les données 2',
),
# 3.py
PYTHON_TEMPLATE.format(
_import='from transifex.native import translate',
call1='translate',
call2='translate',
string1=u'Le canapé 3',
string2=u'Les données 3',
),
]
expected = [
# 1.py
SourceString(
u'Le canapé',
u'désign1,désign2',
_occurrences=['r1/1.py:6'],
),
SourceString(
u'Les données',
u'opération',
_comment='comment',
_tags='t1,t2',
_charlimit=33,
_occurrences=['r1/1.py:7'],
),
# 2.py
SourceString(
u'Le canapé 2',
u'désign1,désign2',
_occurrences=['r1/dir2/2.py:6'],
),
SourceString(
u'Les données 2',
u'opération',
_comment='comment',
_tags='t1,t2',
_charlimit=33,
_occurrences=['r1/dir2/2.py:7'],
),
# 3.py
SourceString(
u'Le canapé 3',
u'désign1,désign2',
_occurrences=['r1/dir3/3.py:6'],
),
SourceString(
u'Les données 3',
u'opération',
_comment='comment',
_tags='t1,t2',
_charlimit=33,
_occurrences=['r1/dir3/3.py:7'],
),
]
compare(expected)