def lint_js_files_are_translated(files_to_lint):
"""Verify that nltext in the js files are marked for translation.
See docstring of: _lint_js_content
Returns:
List of triples: (filename, lineno, error message)
"""
# Make sure jsx files are compiled first, then we will lint the resulting
# js.
kake.make.build_many([
('genfiles/compiled_jsx/en/%s.js' % ka_root.relpath(f), {})
for f in files_to_lint if f.endswith('.jsx')
])
files_to_lint = lintutil.filter(files_to_lint, suffix=('.js', '.jsx'))
for f in files_to_lint:
abs_f = f
f = ka_root.relpath(f)
# Exclude files that we don't need to translate: we don't care
# if those files are 'properly' marked up or not.
if intl.english_only.should_not_translate_file(f):
continue
if f.endswith(".jsx"):
abs_f = "%s/genfiles/compiled_jsx/en/%s.js" % (ka_root.root, f)
f = ka_root.relpath(abs_f)
file_contents = lintutil.file_contents(abs_f)
for error in _lint_js_content(abs_f, file_contents):
yield error
def lint_js_files_are_translated(files_to_lint):
"""Verify that nltext in the js files are marked for translation.
See docstring of: _lint_js_content
Returns:
List of triples: (filename, lineno, error message)
"""
# Make sure jsx files are compiled first, then we will lint the resulting
# js.
kake.make.build_many([('genfiles/compiled_jsx/en/%s.js' %
ka_root.relpath(f), {})
for f in files_to_lint if f.endswith('.jsx')])
files_to_lint = lintutil.filter(files_to_lint, suffix=('.js', '.jsx'))
for f in files_to_lint:
abs_f = f
f = ka_root.relpath(f)
# Exclude files that we don't need to translate: we don't care
# if those files are 'properly' marked up or not.
if intl.english_only.should_not_translate_file(f):
continue
if f.endswith(".jsx"):
abs_f = "%s/genfiles/compiled_jsx/en/%s.js" % (ka_root.root, f)
f = ka_root.relpath(abs_f)
file_contents = lintutil.file_contents(abs_f)
for error in _lint_js_content(abs_f, file_contents):
yield error
def filter(files_to_lint, prefix='', suffix='', exclude_substrings=[]):
"""Return a filtered version of files to lint matching prefix AND suffix.
First it converts each file in files_to_lint to a relative
filename (relative to ka_root). Then it makes sure
relpath.startswith(prefix) and relpath.endswith(suffix).
exclude_substrings is a list: all files which include any
substring in that list is excluded. For exclude_substrings,
the full abspath of the file is considered.
It then converts matching files back to an abspath and returns them.
prefix and suffix can be the same as for startswith and endswith:
either a single string, or a list of strings which are OR-ed
together.
"""
without_excludes = [
f for f in files_to_lint if not any(s in f for s in exclude_substrings)
]
relpaths = [ka_root.relpath(f) for f in without_excludes]
filtered = [
f for f in relpaths if f.startswith(prefix) and f.endswith(suffix)
]
filtered_abspaths = [ka_root.join(f) for f in filtered]
return filtered_abspaths
def lint_strftime(files_to_lint):
"""Complain if you use strftime() instead of i18n.format_date()."""
_BAD_REGEXPS = (
# Javascript
r'toDateString\(\)',
# Jinja2 and python. These are all the modifiers that depend
# on the current locale (e.g. %B).
r'strftime\([\'\"][^\'\"]*%[aAbBcDhpPrxX+]',
# These are modifiers that are numbers, but used in contexts that
# indicate they're probably US-specific, e.g. '%d,', which means
# the current day-of-month followed by a comma, or day before
# month.
r'strftime\([\'\"][^\'\"]*(?:%d,|%d.%m)',
)
bad_re = re.compile('|'.join('(?:%s)' % b for b in _BAD_REGEXPS))
for f in files_to_lint:
relpath = ka_root.relpath(f)
# Ignore third_party code. Normally third_party code wouldn't wind up
# being linted in the first place because all of third_party is in
# webapp's lint_blacklist.txt, but for code that lives in third_party
# that has its own lint_blacklist.txt (e.g. live-editor), webapp's lint
# blacklist.txt gets overridden.
if relpath.startswith('third_party'):
continue
if intl.english_only.should_not_translate_file(relpath):
continue
# Ignore python files we're not uploading to appengine. (We
# can't use this rule with all files since js and html files
# aren't uploaded directly, but we still want to lint them.)
if f.endswith('.py') and _does_not_upload(ka_root.relpath(f)):
continue
badline = lintutil.line_number(f, bad_re, default=None)
if badline is not None:
yield (f, badline,
'Using U.S.-specific date formatting. '
'Use intl.i18n.format_date() and friends instead.')
def lint_strftime(files_to_lint):
"""Complain if you use strftime() instead of i18n.format_date()."""
_BAD_REGEXPS = (
# Javascript
r'toDateString\(\)',
# Jinja2 and python. These are all the modifiers that depend
# on the current locale (e.g. %B).
r'strftime\([\'\"][^\'\"]*%[aAbBcDhpPrxX+]',
# These are modifiers that are numbers, but used in contexts that
# indicate they're probably US-specific, e.g. '%d,', which means
# the current day-of-month followed by a comma, or day before
# month.
r'strftime\([\'\"][^\'\"]*(?:%d,|%d.%m)',
)
bad_re = re.compile('|'.join('(?:%s)' % b for b in _BAD_REGEXPS))
for f in files_to_lint:
relpath = ka_root.relpath(f)
# Ignore third_party code. Normally third_party code wouldn't wind up
# being linted in the first place because all of third_party is in
# webapp's lint_blacklist.txt, but for code that lives in third_party
# that has its own lint_blacklist.txt (e.g. live-editor), webapp's lint
# blacklist.txt gets overridden.
if relpath.startswith('third_party'):
continue
if intl.english_only.should_not_translate_file(relpath):
continue
# Ignore python files we're not uploading to appengine. (We
# can't use this rule with all files since js and html files
# aren't uploaded directly, but we still want to lint them.)
if f.endswith('.py') and _does_not_upload(ka_root.relpath(f)):
continue
badline = lintutil.line_number(f, bad_re, default=None)
if badline is not None:
yield (f, badline, 'Using U.S.-specific date formatting. '
'Use intl.i18n.format_date() and friends instead.')
def lint_all_wsgi_entrypoint_imports(files_to_lint):
"""Enforce that every WSGI entrypoint imports appengine_config first.
On App Engine Standard, appengine_config is guaranteed to be imported
before anything else. But this isn't guaranteed on VM, which is bad
because our code assumes it! So we require every WSGI entrypoint to
manually import it first thing, before any other imports. (It likely only
matters that it's before all first-party and shared imports, but we require
that it be first to keep things simple. We do allow __future__ imports
first, since those are easy to check and python wants them first.)
"""
entrypoint_filenames = set()
for gae_module_name in modules_util.all_modules:
entrypoint_module_names = (
modules_util.app_yaml_entrypoint_modules(gae_module_name))
for entrypoint_module_name in entrypoint_module_names:
filename = importlib.import_module(entrypoint_module_name).__file__
# replace .pyc, .pyo etc.
filename = os.path.splitext(filename)[0] + '.py'
entrypoint_filenames.add(ka_root.relpath(filename))
if lintutil.filter(files_to_lint, suffix='.yaml'):
# If any yaml files were modified, we need to recheck all entrypoints.
# TODO(benkraft): We should really only need to do this if any
# module-yaml, or any yaml included by one, has changed, but
# determining which those are is tricky, so we assume they all are.
filenames_to_check = entrypoint_filenames
else:
# Otherwise, we just need to check changed entrypoints.
files_to_lint = set(ka_root.relpath(f) for f in files_to_lint)
filenames_to_check = entrypoint_filenames & files_to_lint
for filename in filenames_to_check:
if filename.startswith('third_party/'):
continue
maybe_error = _lint_single_wsgi_entrypoint_import(filename)
if maybe_error:
yield maybe_error
def lint_all_wsgi_entrypoint_imports(files_to_lint):
"""Enforce that every WSGI entrypoint imports appengine_config first.
On App Engine Standard, appengine_config is guaranteed to be imported
before anything else. But this isn't guaranteed on VM, which is bad
because our code assumes it! So we require every WSGI entrypoint to
manually import it first thing, before any other imports. (It likely only
matters that it's before all first-party and shared imports, but we require
that it be first to keep things simple. We do allow __future__ imports
first, since those are easy to check and python wants them first.)
"""
entrypoint_filenames = set()
for gae_module_name in modules_util.all_modules:
entrypoint_module_names = (
modules_util.app_yaml_entrypoint_modules(gae_module_name))
for entrypoint_module_name in entrypoint_module_names:
filename = importlib.import_module(entrypoint_module_name).__file__
# replace .pyc, .pyo etc.
filename = os.path.splitext(filename)[0] + '.py'
entrypoint_filenames.add(ka_root.relpath(filename))
if lintutil.filter(files_to_lint, suffix='.yaml'):
# If any yaml files were modified, we need to recheck all entrypoints.
# TODO(benkraft): We should really only need to do this if any
# module-yaml, or any yaml included by one, has changed, but
# determining which those are is tricky, so we assume they all are.
filenames_to_check = entrypoint_filenames
else:
# Otherwise, we just need to check changed entrypoints.
files_to_lint = set(ka_root.relpath(f) for f in files_to_lint)
filenames_to_check = entrypoint_filenames & files_to_lint
for filename in filenames_to_check:
if filename.startswith('third_party/'):
continue
maybe_error = _lint_single_wsgi_entrypoint_import(filename)
if maybe_error:
yield maybe_error
def _jinja2_files():
"""Yield all jinja2 files that might have text-to-translate.
Returned filenames are relative to ka-root.
We assume all .html and .txt files under templates/ is jinja2.
"""
root = ka_root.join('templates')
for (rootdir, dirs, files) in os.walk(root):
reldir = ka_root.relpath(rootdir)
for f in files:
if f.endswith('.html') or f.endswith('.txt'):
relpath = os.path.join(reldir, f)
yield os.path.normpath(relpath)
def _handlebars_files():
"""Yield all handlebars files that might have text-to-translate.
Returned filenames are relative to ka-root.
We assume all .handlebars under javascript needs to be translated. We
have an intl.english_only.should_not_translate check later in this file to
filter out the ones we don't care about.
"""
root = ka_root.join('javascript')
for (rootdir, dirs, files) in os.walk(root):
reldir = ka_root.relpath(rootdir)
for f in files:
if f.endswith('.handlebars'):
relpath = os.path.join(reldir, f)
yield os.path.normpath(relpath)
def lint_circular_imports(files_to_lint):
"""Report on all circular imports found in files_to_lint.
A circular import is when A imports B imports C imports A. We
focus only on *simple* cycles, where each node occurs only once in
a cycle (so a figure-8 would be two cycles, one for the top circle
and one for the bottom, not a single, more complicated cycle).
The rule is that we report all cycles that include any files in
files_to_lint. (For reproducibility, we always print the cycle
starting with the file in the cycle that comes first
alphabetically.)
It's very possible for a single file to have multiple circular
imports. We report them all.
"""
files_to_lint = lintutil.filter(files_to_lint, suffix='.py')
module_to_file = {
ka_root.relpath(os.path.splitext(f)[0]).replace(os.sep, '.'): f
for f in files_to_lint
}
saw_a_cycle = False
for cycle in sorted(_find_all_cycles(module_to_file.keys())):
saw_a_cycle = True
# We report the line-number where cycle[0] imports cycle[1].
f = module_to_file.get(cycle[0])
if f is None: # means this cycle does not include any files-to-lint
continue
import_lines = _get_import_lines(f)
desired_import_line = next(il for il in import_lines
if il.module == cycle[1])
yield (f, desired_import_line.lineno,
('Resolve this import cycle (by making a late import): %s' %
' -> '.join(cycle)))
if not saw_a_cycle:
# This is only safe to do when there are no cycles already!
for error in _lint_unnecessary_late_imports(files_to_lint):
yield error
def _add_import_line(filename, il_dict):
# All legal imports should have an import-part.
assert 'import_part' in il_dict, il_dict
from_part = il_dict.get('from_part')
import_part = il_dict.get('import_part')
as_part = il_dict.get('as_part')
name = as_part if as_part else import_part
# "level" is how many leading dots there are (for a
# relative import like "from ..foo import bar").
module_parts = []
if from_part:
level = len(from_part) - len(from_part.lstrip('.'))
from_name = from_part[level:]
if level > 0:
# Handle relative paths. Each dot is one dir-level up.
filename_parts = ka_root.relpath(filename).split(os.sep)
module_parts.extend(filename_parts[:-level])
if from_name:
module_parts.append(from_name)
else:
level = 0 # only "from" imports can be relative.
module_parts.append(import_part)
module = '.'.join(module_parts)
import_line = ImportLine(filename=filename,
lineno=il_dict['lineno'],
colno=il_dict['colno'],
module=module,
name=name,
level=level,
is_toplevel=il_dict['is_toplevel'],
has_comma=il_dict.get('has_comma', False),
has_from=bool(from_part),
has_as=bool(as_part))
_g_import_line_cache[filename].append(import_line)
def filter(files_to_lint, prefix='', suffix='', exclude_substrings=[]):
"""Return a filtered version of files to lint matching prefix AND suffix.
First it converts each file in files_to_lint to a relative
filename (relative to ka_root). Then it makes sure
relpath.startswith(prefix) and relpath.endswith(suffix).
exclude_substrings is a list: all files which include any
substring in that list is excluded. For exclude_substrings,
the full abspath of the file is considered.
It then converts matching files back to an abspath and returns them.
prefix and suffix can be the same as for startswith and endswith:
either a single string, or a list of strings which are OR-ed
together.
"""
without_excludes = [f for f in files_to_lint
if not any(s in f for s in exclude_substrings)]
relpaths = [ka_root.relpath(f) for f in without_excludes]
filtered = [f for f in relpaths
if f.startswith(prefix) and f.endswith(suffix)]
filtered_abspaths = [ka_root.join(f) for f in filtered]
return filtered_abspaths
def _python_files():
"""Yield all python files that might have text-to-translate.
Returned filenames are relative to ka-root.
We look for all python files under ka-root, ignoring files in
third_party (which we assume handles translation on its own)
and in genfiles/compiled_handlebars_py, which was translated
before being compiled.
"""
for (rootdir, dirs, files) in os.walk(ka_root.root):
# Go backwards so we can erase as we go.
for i in xrange(len(dirs) - 1, -1, -1):
# If we're not a python module, no need to recurse further.
if not os.path.exists(os.path.join(rootdir, dirs[i],
'__init__.py')):
del dirs[i]
elif dirs[i] in ('third_party', 'compiled_handlebars_py'):
del dirs[i]
reldir = ka_root.relpath(rootdir)
for f in files:
if f.endswith('.py') and f != '__init__.py':
yield os.path.normpath(os.path.join(reldir, f))
def lint_missing_used_context_keys(files_to_lint):
"""Attempts to find places the user failed to update used_context_keys().
If you write a compile_rule that uses context['foo'], you're
supposed to advertise that fact by including 'foo' in your
used_context_keys() method. But it's easy to forget to do that.
This rule attempts to remind you by looking at the source code for
your class and trying to find all uses.
This isn't perfect, which is why it's a lint rule and we don't
just automatically extract uses of context['foo'], but it's better
than nothing! If it's claiming a line is a use of context when
it's not, just stick a @Nolint at the end of the line.
"""
# Only files under the kake directory might have compile rules.
relfiles_to_lint = [
ka_root.relpath(f) for f in files_to_lint
if ka_root.relpath(f).startswith('kake/')
]
if not relfiles_to_lint:
return
# This forces us to import all the kake compile_rules.
from kake import make # @UnusedImport
classes = (list(_all_subclasses(compile_rule.CompileBase)) +
list(_all_subclasses(computed_inputs.ComputedInputsBase)))
for cls in classes:
class_file = cls.__module__.replace('.', '/') + '.py'
if class_file not in relfiles_to_lint:
continue
claimed_used_context_keys = set(cls.used_context_keys())
actual_used_context_keys = {} # map from key to linenum where used
class_source = inspect.getsource(cls)
module_source = inspect.getsource(sys.modules[cls.__module__])
# Find what line-number the class we're linting starts on.
class_source_pos = module_source.find(class_source)
class_startline = module_source.count('\n', 0, class_source_pos) + 1
# Find what line-number class.used_context_keys() starts on.
used_context_keys_pos = class_source.find('def used_context_keys')
if used_context_keys_pos == -1:
used_context_keys_line = 1
else:
used_context_keys_line = (
class_source.count('\n', 0, used_context_keys_pos) +
class_startline)
for m in _CONTEXT_USE_RE.finditer(class_source):
if '@Nolint' not in m.group(0):
key = m.group(1) or m.group(2) or m.group(3)
linenum = (class_source.count('\n', 0, m.start()) +
class_startline)
actual_used_context_keys.setdefault(key, linenum)
must_add = set(actual_used_context_keys) - claimed_used_context_keys
must_remove = claimed_used_context_keys - set(actual_used_context_keys)
for key in must_add:
# We don't require people to register system keys (start with _)
# or glob vars (start with '{').
if not key.startswith(('_', '{')):
yield (
ka_root.join(class_file),
actual_used_context_keys[key], # linenum
'Build rule uses "%s" but it is not listed in'
' used_context_keys(). Add it there or mark'
' it with @Nolint if this is in error.' % key)
for key in must_remove:
yield (ka_root.join(class_file), used_context_keys_line,
'Build rule does not use "%s" but it is listed in'
' used_context_keys(). Add it there or fix this'
' linter if it is in error.' % key)
def lint_missing_used_context_keys(files_to_lint):
"""Attempts to find places the user failed to update used_context_keys().
If you write a compile_rule that uses context['foo'], you're
supposed to advertise that fact by including 'foo' in your
used_context_keys() method. But it's easy to forget to do that.
This rule attempts to remind you by looking at the source code for
your class and trying to find all uses.
This isn't perfect, which is why it's a lint rule and we don't
just automatically extract uses of context['foo'], but it's better
than nothing! If it's claiming a line is a use of context when
it's not, just stick a @Nolint at the end of the line.
"""
# Only files under the kake directory might have compile rules.
relfiles_to_lint = [ka_root.relpath(f) for f in files_to_lint
if ka_root.relpath(f).startswith('kake/')]
if not relfiles_to_lint:
return
# This forces us to import all the kake compile_rules.
from kake import make # @UnusedImport
classes = (list(_all_subclasses(compile_rule.CompileBase)) +
list(_all_subclasses(computed_inputs.ComputedInputsBase)))
for cls in classes:
class_file = cls.__module__.replace('.', '/') + '.py'
if class_file not in relfiles_to_lint:
continue
claimed_used_context_keys = set(cls.used_context_keys())
actual_used_context_keys = {} # map from key to linenum where used
class_source = inspect.getsource(cls)
module_source = inspect.getsource(sys.modules[cls.__module__])
# Find what line-number the class we're linting starts on.
class_source_pos = module_source.find(class_source)
class_startline = module_source.count('\n', 0, class_source_pos) + 1
# Find what line-number class.used_context_keys() starts on.
used_context_keys_pos = class_source.find('def used_context_keys')
if used_context_keys_pos == -1:
used_context_keys_line = 1
else:
used_context_keys_line = (class_source.count('\n', 0,
used_context_keys_pos)
+ class_startline)
for m in _CONTEXT_USE_RE.finditer(class_source):
if '@Nolint' not in m.group(0):
key = m.group(1) or m.group(2) or m.group(3)
linenum = (class_source.count('\n', 0, m.start())
+ class_startline)
actual_used_context_keys.setdefault(key, linenum)
must_add = set(actual_used_context_keys) - claimed_used_context_keys
must_remove = claimed_used_context_keys - set(actual_used_context_keys)
for key in must_add:
# We don't require people to register system keys (start with _)
# or glob vars (start with '{').
if not key.startswith(('_', '{')):
yield (ka_root.join(class_file),
actual_used_context_keys[key], # linenum
'Build rule uses "%s" but it is not listed in'
' used_context_keys(). Add it there or mark'
' it with @Nolint if this is in error.' % key)
for key in must_remove:
yield (ka_root.join(class_file),
used_context_keys_line,
'Build rule does not use "%s" but it is listed in'
' used_context_keys(). Add it there or fix this'
' linter if it is in error.' % key)
def lint_templates_are_translated(files_to_lint):
"""Verify that nltext in the input templates are marked for translation.
All natural-language text in jinja2 and handlebars files should be
marked for translation, using {{ _("...") }} or {{#_}}...{{/_}}.
i18nize_templates.py is a tool that can do this for you automatically.
We run this tool in 'check' mode to verify that every input file
is already marked up appropriately.
Since i18nize_templates isn't perfect (it thinks you need to
translate text like 'Lebron James' or 'x' when used on a 'close'
button), you can use nolint-like functionality to tell this linter
it's ok if some text is not marked up to be translated. Unlike
other tests though, we do not use the @Nolint directive for this,
but instead wrap the relevant text in
{{ i18n_do_not_translate(...) }}
or
{{#i18nDoNotTranslate}}...{{/i18nDoNotTranslate}}
"""
# Add some ka-specific function we know do not have nltext arguments.
i18nize_templates.mark_function_args_lack_nltext(
'js_css_packages.package',
'js_css_packages.script',
'handlebars_template',
'youtube.player_embed',
'log.date.strftime',
'emails.tracking_image_url',
'templatetags.to_canonical_url',
'render_react',
)
for f in files_to_lint:
abs_f = f
f = ka_root.relpath(f)
# Exclude files that we don't need to translate: we don't care
# if those files are 'properly' marked up or not.
if intl.english_only.should_not_translate_file(f):
continue
if (f.startswith('templates' + os.sep) and
(f.endswith('.html') or f.endswith('.txt'))):
# jinja2 template
parser = i18nize_templates.get_parser_for_file(f)
correction = 'wrap the text in {{ i18n_do_not_translate() }}'
elif f.endswith('.handlebars'):
# handlebars template
parser = i18nize_templates.get_parser_for_file(f)
correction = ('wrap the text in {{#i18nDoNotTranslate}}...'
'{{/i18nDoNotTranslate}}')
else:
continue
file_contents = lintutil.file_contents(abs_f)
try:
parsed_output = parser.parse(
file_contents.decode('utf-8')).encode('utf-8')
except i18nize_templates.HTMLParser.HTMLParseError, why:
m = re.search(r'at line (\d+)', str(why))
linenum = int(m.group(1)) if m else 1
yield (abs_f, linenum,
'"i18nize_templates.py %s" fails: %s' % (f, why))
continue
orig_lines = file_contents.splitlines()
parsed_lines = parsed_output.splitlines()
for i in xrange(len(orig_lines)):
if orig_lines[i] != parsed_lines[i]:
yield (abs_f, i + 1,
'Missing _(); run tools/i18nize_templates.py or %s '
'(expecting "%s")'
% (correction, parsed_lines[i].strip()))
def lint_not_using_gettext_at_import_time(files_to_lint):
"""Make sure we don't use i18n._/etc in a static context.
If you have a global variable such as '_FOO = i18n._("bar")', at
the top of some .py file, it won't work the way you intend because
i18n._() needs to be called while handling a request in order to
know what language to translate to. (Instead, you'd need to do
_FOO = lambda: i18n._("bar")
or some such.)
This tests for this by mocking i18n._ et al., and then importing
everything (but running nothing). Any i18n._ calls that happen
during this import are problematic! We have to spawn a new
python process to make sure we do the importing properly (and
without messing with the currently running python environment!)
"""
candidate_files_to_lint = lintutil.filter(files_to_lint, suffix='.py')
files_to_lint = []
for filename in candidate_files_to_lint:
contents = lintutil.file_contents(filename)
# Check that it's plausible this file uses i18n._ or similar.
# This also avoids importing random third-party files that may
# have nasty side-effects at import time (all our code is too
# well-written to do that!)
if 'import intl' in contents or 'from intl' in contents:
files_to_lint.append(filename)
program = """\
import os # @Nolint(linter can't tell this is in a string!)
import sys # @Nolint(linter can't tell this is in a string!)
import traceback
import intl.request # @Nolint(seems unused to our linter but it's used)
_ROOT = "%s"
def add_lint_error(f):
# We assume code in 'intl' doesn't make this mistake, and thus
# the first stack-frame before we get into 'intl' is the
# offending code. ctx == '<string>' means the error occurred in
# this pseudo-script.
for (ctx, lineno, fn, line) in reversed(traceback.extract_stack()):
if os.path.isabs(ctx):
ctx = os.path.relpath(ctx, _ROOT)
if ctx != '<string>' and not ctx.startswith('intl/'):
if ctx == f:
print 'GETTEXT ERROR {} {}'.format(ctx, lineno)
break
return 'en' # a fake value for intl.request.ka_locale
""" % ka_root.root
if not files_to_lint:
return
for filename in files_to_lint:
modulename = ka_root.relpath(filename)
modulename = os.path.splitext(modulename)[0] # nix .py
modulename = modulename.replace('/', '.')
# Force a re-import.
program += 'sys.modules.pop("%s", None)\n' % modulename
program += ('intl.request.ka_locale = lambda: add_lint_error("%s")\n' %
ka_root.relpath(filename))
program += 'import %s\n' % modulename
p = subprocess.Popen([
'env',
'PYTHONPATH=%s' % ':'.join(sys.path), sys.executable, '-c', program
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
p.wait()
lint_output = p.stdout.read()
for line in lint_output.splitlines():
if line.startswith('GETTEXT ERROR '):
line = line[len('GETTEXT ERROR '):]
(filename, linenum) = line.rsplit(' ', 1)
yield (ka_root.join(filename), int(linenum),
'Trying to translate at import-time, but '
'translation only works at runtime! '
'Use intl.i18n.mark_for_translation() instead.')
def lint_templates_are_translated(files_to_lint):
"""Verify that nltext in the input templates are marked for translation.
All natural-language text in jinja2 and handlebars files should be
marked for translation, using {{ _("...") }} or {{#_}}...{{/_}}.
i18nize_templates.py is a tool that can do this for you automatically.
We run this tool in 'check' mode to verify that every input file
is already marked up appropriately.
Since i18nize_templates isn't perfect (it thinks you need to
translate text like 'Lebron James' or 'x' when used on a 'close'
button), you can use nolint-like functionality to tell this linter
it's ok if some text is not marked up to be translated. Unlike
other tests though, we do not use the @Nolint directive for this,
but instead wrap the relevant text in
{{ i18n_do_not_translate(...) }}
or
{{#i18nDoNotTranslate}}...{{/i18nDoNotTranslate}}
"""
# Add some ka-specific function we know do not have nltext arguments.
i18nize_templates.mark_function_args_lack_nltext(
'js_css_packages.package',
'js_css_packages.script',
'handlebars_template',
'youtube.player_embed',
'log.date.strftime',
'emails.tracking_image_url',
'templatetags.to_canonical_url',
'render_react',
)
for f in files_to_lint:
abs_f = f
f = ka_root.relpath(f)
# Exclude files that we don't need to translate: we don't care
# if those files are 'properly' marked up or not.
if intl.english_only.should_not_translate_file(f):
continue
if (f.startswith('templates' + os.sep)
and (f.endswith('.html') or f.endswith('.txt'))):
# jinja2 template
parser = i18nize_templates.get_parser_for_file(f)
correction = 'wrap the text in {{ i18n_do_not_translate() }}'
elif f.endswith('.handlebars'):
# handlebars template
parser = i18nize_templates.get_parser_for_file(f)
correction = ('wrap the text in {{#i18nDoNotTranslate}}...'
'{{/i18nDoNotTranslate}}')
else:
continue
file_contents = lintutil.file_contents(abs_f)
try:
parsed_output = parser.parse(
file_contents.decode('utf-8')).encode('utf-8')
except i18nize_templates.HTMLParser.HTMLParseError, why:
m = re.search(r'at line (\d+)', str(why))
linenum = int(m.group(1)) if m else 1
yield (abs_f, linenum,
'"i18nize_templates.py %s" fails: %s' % (f, why))
continue
orig_lines = file_contents.splitlines()
parsed_lines = parsed_output.splitlines()
for i in xrange(len(orig_lines)):
if orig_lines[i] != parsed_lines[i]:
yield (abs_f, i + 1,
'Missing _(); run tools/i18nize_templates.py or %s '
'(expecting "%s")' %
(correction, parsed_lines[i].strip()))
def lint_not_using_gettext_at_import_time(files_to_lint):
"""Make sure we don't use i18n._/etc in a static context.
If you have a global variable such as '_FOO = i18n._("bar")', at
the top of some .py file, it won't work the way you intend because
i18n._() needs to be called while handling a request in order to
know what language to translate to. (Instead, you'd need to do
_FOO = lambda: i18n._("bar")
or some such.)
This tests for this by mocking i18n._ et al., and then importing
everything (but running nothing). Any i18n._ calls that happen
during this import are problematic! We have to spawn a new
python process to make sure we do the importing properly (and
without messing with the currently running python environment!)
"""
candidate_files_to_lint = lintutil.filter(files_to_lint, suffix='.py')
files_to_lint = []
for filename in candidate_files_to_lint:
contents = lintutil.file_contents(filename)
# Check that it's plausible this file uses i18n._ or similar.
# This also avoids importing random third-party files that may
# have nasty side-effects at import time (all our code is too
# well-written to do that!)
if 'import intl' in contents or 'from intl' in contents:
files_to_lint.append(filename)
program = """\
import os # @Nolint(linter can't tell this is in a string!)
import sys # @Nolint(linter can't tell this is in a string!)
import traceback
import intl.request # @Nolint(seems unused to our linter but it's used)
_ROOT = "%s"
def add_lint_error(f):
# We assume code in 'intl' doesn't make this mistake, and thus
# the first stack-frame before we get into 'intl' is the
# offending code. ctx == '<string>' means the error occurred in
# this pseudo-script.
for (ctx, lineno, fn, line) in reversed(traceback.extract_stack()):
if os.path.isabs(ctx):
ctx = os.path.relpath(ctx, _ROOT)
if ctx != '<string>' and not ctx.startswith('intl/'):
if ctx == f:
print 'GETTEXT ERROR {} {}'.format(ctx, lineno)
break
return 'en' # a fake value for intl.request.ka_locale
""" % ka_root.root
if not files_to_lint:
return
for filename in files_to_lint:
modulename = ka_root.relpath(filename)
modulename = os.path.splitext(modulename)[0] # nix .py
modulename = modulename.replace('/', '.')
# Force a re-import.
program += 'sys.modules.pop("%s", None)\n' % modulename
program += ('intl.request.ka_locale = lambda: add_lint_error("%s")\n'
% ka_root.relpath(filename))
program += 'import %s\n' % modulename
p = subprocess.Popen(
['env', 'PYTHONPATH=%s' % ':'.join(sys.path),
sys.executable, '-c', program],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
p.wait()
lint_output = p.stdout.read()
for line in lint_output.splitlines():
if line.startswith('GETTEXT ERROR '):
line = line[len('GETTEXT ERROR '):]
(filename, linenum) = line.rsplit(' ', 1)
yield (ka_root.join(filename), int(linenum),
'Trying to translate at import-time, but '
'translation only works at runtime! '
'Use intl.i18n.mark_for_translation() instead.')
def _lint_unnecessary_late_imports(files_to_lint):
"""Report on all late imports that could safely be top-level.
We make an import "late" -- that is, inside a function rather than
at the top level -- when it's needed to avoid circular imports.
However, sometimes a refactor makes it so an import doesn't need to
be late anymore. That's hard to tell by inspection, so this linter
does it for you!
Conceptually, this is a standalone linter, so I wrote it like that.
However, it doesn't deal well with cycles so we only want to run it
when there are no cycles. Thus, we run it at the end of the
cycle-checking linter, and then only if there are no cycles.
"""
files_to_lint = lintutil.filter(files_to_lint, suffix='.py')
# Files we shouldn't check for late imports for some reason.
# Can end with `.` to match all modules starting with this prefix.
MODULE_BLACKLIST = frozenset([
# Is imported first, must do minimal work
'appengine_config',
# Minimize deps so new code can import it without fear.
'users.current_user',
# Minimize deps to keep `make tesc` output small.
'testutil.gae_model',
'testutil.mapreduce_stub',
# Does a bunch of importing after fixing up sys.path
'tools.appengine_tool_setup',
'tools.devservercontext',
'tools.devshell',
# TODO(csilvers): remove this and move the babel routines to their
# own file instead.
'shared_jinja',
# Does importing after modifying modules.
'pickle_util_test',
# Has its own style rule stating late imports are preferred
'kake.',
])
# Imports that are allowed to be late for some reason.
LATE_IMPORT_BLACKLIST = frozenset([
# Can only import once we've verified we're not running in prod
'sandbox_util',
'mock',
'kake.server_client',
'kake.make',
])
for f in files_to_lint:
module = ka_root.relpath(os.path.splitext(f)[0]).replace(os.sep, '.')
# Some files are expected to have late imports.
if (module in MODULE_BLACKLIST or module.startswith(
tuple(m for m in MODULE_BLACKLIST if m.endswith('.')))):
continue
for late_import in _get_late_imports(module):
# Some modules can *only* be late-imported,
if late_import in LATE_IMPORT_BLACKLIST:
continue
# If late_import transitively imports us, then it's not safe
# to move it to the top level: that would introduce a cycle.
if module in _calculate_transitive_imports(late_import):
continue
# If a module is marked `@UnusedImport` or `@Nolint`, then
# it's being imported for its side effects, and we don't
# want to move it to the top level.
import_lines = _get_import_lines(f)
late_import_line = next(
il for il in import_lines
if (il.module == late_import and not il.is_toplevel))
contents = lintutil.file_contents(f)
lines = contents.splitlines(True)
if ('@UnusedImport' in lines[late_import_line.lineno - 1]
or '@Nolint' in lines[late_import_line.lineno - 1]):
continue
# If we get here, it's safe to move this import to the top level!
yield (f, late_import_line.lineno,
"Make this a top-level import: it doesn't cause cycles")
# For the rest of our analysis, assume that `module` has
# moved the import of `late_import` to the top level.
# TODO(csilvers): figure out the "best" edge to add, rather
# than just doing first come first served.
_add_edge(module, late_import)
