def handle(self, *args, **options):
self.domain = options['domain']
self.verbosity = options['verbosity']
self.ignore_patterns = []
self.purge = options['purge']
self.symlinks = options['symlinks']
extensions = options['extensions']
if self.domain == 'djangojs':
exts = extensions if extensions else ['js']
else:
exts = extensions if extensions else ['html', 'txt', 'py']
self.extensions = handle_extensions(exts)
self.locale_paths = []
self.default_locale_path = None
self.string_collection = SourceStringCollection()
# Create an extractor for Python files, to reuse for all files
self.python_extractor = Extractor()
# Support `t()` and `ut()` calls made on the Django module.
self.python_extractor.register_functions(
'transifex.native.django.t', 'transifex.native.django.ut',
'transifex.native.django.lazyt')
self.stats = {'processed_files': 0, 'strings': []}
# Search all related files and collect translatable strings,
# storing them in `self.string_collection`
self.collect_strings()
# Push the strings to the CDS
self.push_strings()
def test_exceptions_on_import(self):
src = TEMPLATE.format(
_import='from native import 33',
call1='_',
call2='_',
)
ex = Extractor()
results = ex.extract_strings(src, 'myfile.py')
assert results == []
assert ex.errors[0][0] == 'myfile.py'
assert isinstance(ex.errors[0][1], SyntaxError)
def test_exceptions_on_function_call(self):
src = TEMPLATE.format(
_import='from transifex.native import translate',
call1='33', # should produce an error
call2='_',
)
ex = Extractor()
results = ex.extract_strings(src, 'myfile.py')
assert results == []
assert ex.errors[0][0] == 'myfile.py'
assert isinstance(ex.errors[0][1], AttributeError)
# Num/Constant discrepancy between python versions
assert re.search(
re.escape("Invalid module/function format on line 6 col 0: '") +
r'Num|Constant' + re.escape("' object has no attribute 'attr'"),
ex.errors[0][1].args[0])
def test_registered_imports(self):
# Test all combinations in multi-level imports
# with a custom registered function
ex = Extractor()
ex.register_functions('module1.module2.module3.myfunc')
src = TEMPLATE.format(
_import=('from module1.module2 import module3 as m3\n'
'import module1.module2.module3 as _m3\n'),
call1='m3.myfunc',
call2='_m3.myfunc',
)
results = ex.extract_strings(src, 'myfile.py')
assert results == self._strings()
src = TEMPLATE.format(
_import=('from module1 import module2 as m2\n'
'import module1.module2 as _m2\n'),
call1='m2.module3.myfunc',
call2='_m2.module3.myfunc',
)
results = ex.extract_strings(src, 'myfile.py')
assert results == self._strings()
src = TEMPLATE.format(
_import='import module1 as _m1',
call1='_m1.module2.module3.myfunc',
call2='_m1.module2.module3.myfunc',
)
# mocking gets a little bit different here shifting occurrences
expected = [
SourceString(u'Le canapé',
u'désign',
param1='1',
param2=2,
param3=True,
_occurrences=['myfile.py:6']),
SourceString(u'Les données',
u'opération',
_comment='comment',
_tags=['t1', 't2'],
_charlimit=33,
_occurrences=['myfile.py:7']),
]
results = ex.extract_strings(src, 'myfile.py')
assert results == expected
class Push(CommandMixin):
def add_arguments(self, subparsers):
parser = subparsers.add_parser(
'push',
help=("Detect translatable strings in Django templates and Python "
"files and push them as source strings to Transifex"),
)
parser.add_argument(
'--extension',
'-e',
dest='extensions',
action='append',
help=('The file extension(s) to examine (default: "html,txt,py", '
'or "js" if the domain is "djangojs"). Separate multiple '
'extensions with commas, or use -e multiple times.'),
)
parser.add_argument(
'--purge',
'-p',
action='store_true',
dest='purge',
default=False,
help=('Replace the entire resource content with the '
'pushed content of this request. If not provided (the '
'default), then append the source content of this request '
'to the existing resource content.'),
)
parser.add_argument(
'--symlinks',
'-s',
action='store_true',
dest='symlinks',
default=False,
help=('Follows symlinks to directories when examining source code '
'and templates for translation strings.'),
)
def handle(self, *args, **options):
self.domain = options['domain']
self.verbosity = options['verbosity']
self.ignore_patterns = []
self.purge = options['purge']
self.symlinks = options['symlinks']
extensions = options['extensions']
if self.domain == 'djangojs':
exts = extensions if extensions else ['js']
else:
exts = extensions if extensions else ['html', 'txt', 'py']
self.extensions = handle_extensions(exts)
self.locale_paths = []
self.default_locale_path = None
self.string_collection = SourceStringCollection()
# Create an extractor for Python files, to reuse for all files
self.python_extractor = Extractor()
# Support `t()` and `ut()` calls made on the Django module.
self.python_extractor.register_functions(
'transifex.native.django.t', 'transifex.native.django.ut',
'transifex.native.django.lazyt')
self.stats = {'processed_files': 0, 'strings': []}
# Search all related files and collect translatable strings,
# storing them in `self.string_collection`
self.collect_strings()
# Push the strings to the CDS
self.push_strings()
def collect_strings(self):
"""Search all related files, collect and store translatable strings.
Stores found strings in `self.string_collection`.
"""
Color.echo(
'[high]\n'
'##############################################################\n'
'Transifex Native: Parsing files to detect translatable content'
'[end]')
files = self._find_files('.', 'push')
for f in files:
extracted = self._extract_strings(f)
self.string_collection.extend(extracted)
self.stats['processed_files'] += 1
if extracted and len(extracted):
self.stats['strings'].append((f.file, len(extracted)))
self._show_collect_results()
def push_strings(self):
"""Push strings to the CDS."""
total = len(self.string_collection.strings)
if total == 0:
Color.echo('[warn]There are no strings to push to Transifex.[end]')
return
Color.echo('Pushing [warn]{}[end] unique translatable strings '
'to Transifex...'.format(total))
status_code, response_content = tx.push_source_strings(
self.string_collection.strings.values(), self.purge)
self._show_push_results(status_code, response_content)
def _extract_strings(self, translatable_file):
"""Extract source strings from the given file.
Supports both Python files and Django template files.
:param TranslatableFile translatable_file: the file to search
:return: a list of SourceString objects
:rtype: list
"""
self.verbose('Processing file %s in %s' %
(translatable_file.file, translatable_file.dirpath))
encoding = (settings.FILE_CHARSET
if self.settings_available else 'utf-8')
try:
src_data = self._read_file(translatable_file.path, encoding)
except UnicodeDecodeError as e:
self.verbose(
'UnicodeDecodeError: skipped file %s in %s (reason: %s)' % (
translatable_file.file,
translatable_file.dirpath,
e,
))
return None
_, extension = os.path.splitext(translatable_file.file)
# Python file
if extension == '.py':
return self.python_extractor.extract_strings(
force_text(src_data), translatable_file.path[2:])
# Template file
return extract_transifex_template_strings(
src_data,
translatable_file.path[2:],
encoding,
)
def _show_collect_results(self):
"""Display results of collecting source strings from files."""
total_strings = sum([x[1] for x in self.stats['strings']])
Color.echo('Processed [warn]{}[end] files and found [warn]{}[end] '
'translatable strings in [warn]{}[end] of them.'.format(
self.stats['processed_files'], total_strings,
len(self.stats)))
def _show_push_results(self, status_code, response_content):
"""Display results of pushing the source strings to CDS.
:param int status_code: the HTTP status code
:param dict response_content: the content of the response
"""
try:
if 200 <= status_code < 300:
# {"created":0,"updated":5,"skipped":1,"deleted":0,"failed":0,"errors":[]}
created = response_content.get('created')
updated = response_content.get('updated')
skipped = response_content.get('skipped')
deleted = response_content.get('deleted')
failed = response_content.get('failed')
errors = response_content.get('errors', [])
Color.echo(
'[green]\nSuccessfully pushed strings to Transifex.[end]\n'
'[high]Status:[end] [warn]{code}[end]\n'
'[high]Created strings:[end] [warn]{created}[end]\n'
'[high]Updated strings:[end] [warn]{updated}[end]\n'
'[high]Skipped strings:[end] [warn]{skipped}[end]\n'
'[high]Deleted strings:[end] [warn]{deleted}[end]\n'
'[high]Failed strings:[end] [warn]{failed}[end]\n'
'[high]Errors:[end] {errors}[end]\n'.format(
code=status_code,
created=created,
updated=updated,
skipped=skipped,
deleted=deleted,
failed=failed,
errors='\n'.join(errors)))
else:
message = response_content.get('message')
details = response_content.get('details')
Color.echo(
'[error]\nCould not push strings to Transifex.[end]\n'
'[high]Status:[end] [warn]{code}[end]\n'
'[high]Message:[end] [warn]{message}[end]\n'
'[high]Details:[end] [warn]{details}[end]\n'.format(
code=status_code,
message=message,
details=json.dumps(details, indent=4),
))
except Exception:
self.output('(Error while printing formatted report, '
'falling back to raw format)\n'
'Status: {code}\n'
'Content: {content}'.format(
code=status_code,
content=response_content,
))
def _assert(self, src):
ex = Extractor()
results = ex.extract_strings(src, 'myfile.py')
return results == self._strings()
class Push(CommandMixin):
def add_arguments(self, subparsers):
parser = subparsers.add_parser(
'push',
help=("Detect translatable strings in Django templates and Python "
"files and push them as source strings to Transifex"),
)
parser.add_argument(
'--extension',
'-e',
dest='extensions',
action='append',
help=('The file extension(s) to examine (default: "html,txt,py", '
'or "js" if the domain is "djangojs"). Separate multiple '
'extensions with commas, or use -e multiple times.'),
)
parser.add_argument(
'--purge',
'-p',
action='store_true',
dest='purge',
default=False,
help=('Replace the entire resource content with the '
'pushed content of this request. If not provided (the '
'default), then append the source content of this request '
'to the existing resource content.'),
)
parser.add_argument(
'--append-tags',
dest='append_tags',
default=None,
help=('Append tags to strings when pushing to Transifex'),
)
parser.add_argument(
'--with-tags-only',
dest='with_tags_only',
default=None,
help=('Push only strings that contain specific tags'),
)
parser.add_argument(
'--without-tags-only',
dest='without_tags_only',
default=None,
help=('Push only strings that do not contain specific tags'),
)
parser.add_argument(
'--dry-run',
action='store_true',
dest='dry_run',
default=False,
help=('Do not push to CDS'),
)
parser.add_argument(
'--no-wait',
action='store_true',
dest='no_wait',
default=False,
help=('Disable polling for upload results'),
)
parser.add_argument(
'--verbose',
'-v',
action='store_true',
dest='verbose_output',
default=False,
help=('Verbose output'),
)
parser.add_argument(
'--symlinks',
'-s',
action='store_true',
dest='symlinks',
default=False,
help=('Follows symlinks to directories when examining source code '
'and templates for translation strings.'),
)
parser.add_argument(
'--key-generator',
dest='key_generator',
default='source',
choices=['source', 'hash'],
help=('Use "hash" or "source" based keys (default: source)'),
)
def handle(self, *args, **options):
self.verbose_output = options['verbose_output']
self.domain = options['domain']
self.ignore_patterns = []
self.purge = options['purge']
self.symlinks = options['symlinks']
self.append_tags = options['append_tags']
self.with_tags_only = options['with_tags_only']
self.without_tags_only = options['without_tags_only']
self.dry_run = options['dry_run']
self.no_wait = options['no_wait']
self.key_generator = options['key_generator']
extensions = options['extensions']
if self.domain == 'djangojs':
exts = extensions if extensions else ['js']
else:
exts = extensions if extensions else ['html', 'txt', 'py']
self.extensions = handle_extensions(exts)
self.locale_paths = []
self.default_locale_path = None
self.string_collection = SourceStringCollection()
# Create an extractor for Python files, to reuse for all files
self.python_extractor = Extractor()
# Support `t()` and `ut()` calls made on the Django module.
self.python_extractor.register_functions(
'transifex.native.django.t', 'transifex.native.django.ut',
'transifex.native.django.lazyt')
self.stats = {'processed_files': 0, 'strings': []}
# Search all related files and collect translatable strings,
# storing them in `self.string_collection`
self.collect_strings()
# Push the strings to the CDS
if not self.dry_run:
self.push_strings()
def collect_strings(self):
"""Search all related files, collect and store translatable strings.
Stores found strings in `self.string_collection`.
"""
Color.echo(
'[high]\n'
'##############################################################\n'
'Transifex Native: Parsing files to detect translatable content'
'[end]')
files = self._find_files('.', 'push')
for f in files:
extracted_strings = self._extract_strings(f)
self.string_collection.extend(extracted_strings)
self.stats['processed_files'] += 1
if extracted_strings and len(extracted_strings):
self.stats['strings'].append((f.file, len(extracted_strings)))
# Append optional CLI tags
if self.append_tags:
extra_tags = [x.strip() for x in self.append_tags.split(',')]
for key, string in self.string_collection.strings.items():
new_string_tags = set(string.tags + extra_tags)
string.meta[consts.KEY_TAGS] = list(new_string_tags)
# Filter out strings based on tags, i.e. only push strings
# that contain certain tags or do not contain certain tags
if self.with_tags_only:
included_tags = {x.strip() for x in self.with_tags_only.split(',')}
else:
included_tags = set()
if self.without_tags_only:
excluded_tags = {
x.strip()
for x in self.without_tags_only.split(',')
}
else:
excluded_tags = set()
if included_tags or excluded_tags:
self.string_collection.update([
string
for key, string in self.string_collection.strings.items()
if included_tags.issubset(set(string.tags))
and not excluded_tags.intersection(set(string.tags))
])
self._show_collect_results()
def push_strings(self):
"""Push strings to the CDS."""
total = len(self.string_collection.strings)
if total == 0:
Color.echo('[warn]There are no strings to push to Transifex.[end]')
return
Color.echo('Pushing [warn]{}[end] unique translatable strings '
'to Transifex...'.format(total))
status_code, response_content = tx.push_source_strings(
self.string_collection.strings.values(), self.purge)
if self.no_wait:
Color.echo('Queued')
return
job_url = response_content['data']['links']['job']
status = 'starting'
while status in ['starting', 'pending', 'processing']:
time.sleep(1)
status_code, response_content = tx.get_push_status(job_url)
new_status = response_content['data']['status']
if new_status != status:
status = new_status
if status == 'pending':
sys.stdout.write('In queue...')
elif status == 'processing':
sys.stdout.write('Processing...')
else:
sys.stdout.write('.')
sys.stdout.flush()
Color.echo('')
self._show_push_results(status_code, response_content)
def _extract_strings(self, translatable_file):
"""Extract source strings from the given file.
Supports both Python files and Django template files.
:param TranslatableFile translatable_file: the file to search
:return: a list of SourceString objects
:rtype: list
"""
fkeygen = generate_key
if self.key_generator == 'hash':
fkeygen = generate_hashed_key
self.verbose('Processing file %s in %s' %
(translatable_file.file, translatable_file.dirpath))
encoding = (settings.FILE_CHARSET if
(hasattr(settings, 'FILE_CHARSET')
and self.settings_available) else 'utf-8')
try:
src_data = self._read_file(translatable_file.path, encoding)
except UnicodeDecodeError as e:
self.verbose(
'UnicodeDecodeError: skipped file %s in %s (reason: %s)' % (
translatable_file.file,
translatable_file.dirpath,
e,
))
return None
_, extension = os.path.splitext(translatable_file.file)
# Python file
if extension == '.py':
return self.python_extractor.extract_strings(
force_text(src_data), translatable_file.path[2:], fkeygen)
# Template file
return extract_transifex_template_strings(src_data,
translatable_file.path[2:],
encoding, fkeygen)
def _show_collect_results(self):
"""Display results of collecting source strings from files."""
total_strings = sum([x[1] for x in self.stats['strings']])
Color.echo('Processed [warn]{}[end] files and found [warn]{}[end] '
'translatable strings in [warn]{}[end] of them.'.format(
self.stats['processed_files'], total_strings,
len(self.stats)))
if self.verbose_output:
file_list = '\n'.join([
u'[pink]{}.[end] {}'.format((cnt + 1), string_repr(x)) for cnt,
x in enumerate(self.string_collection.strings.values())
])
Color.echo(file_list)
def _show_push_results(self, status_code, response_content):
"""Display results of pushing the source strings to CDS.
:param int status_code: the HTTP status code
:param dict response_content: the content of the response
"""
try:
data = response_content.get('data')
status = data.get('status')
errors = data.get('errors', [])
if status == 'completed':
details = data.get('details')
created = details.get('created')
updated = details.get('updated')
skipped = details.get('skipped')
deleted = details.get('deleted')
failed = details.get('failed')
Color.echo(
'[green]\nSuccessfully pushed strings to Transifex.[end]')
if created > 0:
Color.echo('[high]Created strings:[end] '
'[warn]{created}[end]'.format(created=created))
if updated > 0:
Color.echo('[high]Updated strings:[end] '
'[warn]{updated}[end]'.format(updated=updated))
if skipped > 0:
Color.echo('[high]Skipped strings:[end] '
'[warn]{skipped}[end]'.format(skipped=skipped))
if deleted > 0:
Color.echo('[high]Deleted strings:[end] '
'[warn]{deleted}[end]'.format(deleted=deleted))
if failed > 0:
Color.echo('[high]Failed strings:[end] '
'[warn]{failed}[end]'.format(failed=failed))
else:
Color.echo(
'[error]\nCould not push strings to Transifex.[end]')
if len(errors) > 0:
Color.echo('[high]Errors:[end] {errors}[end]\n'.format(
errors='\n'.join(errors)))
except Exception:
self.output('(Error while printing formatted report, '
'falling back to raw format)\n'
'Status: {code}\n'
'Content: {content}'.format(
code=status_code,
content=response_content,
))