def test_actual_react_code(self):
contents = """\
import React, {Component} from "react";
export class Dashboard extends Component {
render() {
return <div data-test-id="coach-dashboard">Should translate!</div>;
}
};
"""
kake.make.build('genfiles/node_modules/babel-core/package.json')
tmpdir = os.path.realpath(
tempfile.mkdtemp(prefix=(self.__class__.__name__ + '.')))
try:
# We make our tmpdir look like genfiles.
for f in os.listdir(ka_root.join('genfiles')):
os.symlink(ka_root.join('genfiles', f),
os.path.join(tmpdir, f))
fname = ka_root.join(tmpdir, 'bad.jsx')
with open(fname, 'w') as f:
f.write(contents)
p = subprocess.Popen(
['node', ka_root.join('kake', 'compile_js.js')],
stdin=subprocess.PIPE,
cwd=ka_root.root)
p.communicate(input=json.dumps([[fname, fname + '.js']]))
self.assertEqual(0, p.returncode)
with open(fname + '.js') as f:
compiled_jsx = f.read()
self.assert_error(compiled_jsx)
finally:
shutil.rmtree(tmpdir)
def _lint(self, contents):
self.set_file_contents(ka_root.join('test.py'), contents)
for (file, line, msg) in import_lint.lint_unused_and_missing_imports(
[ka_root.join('test.py')]):
# We ignore unused imports; we have a different test for that.
if msg.startswith('Missing import:'):
yield (file, line, msg)
def test_symlinks_update_when_args_change(self):
self._build(['js'],
readable=True,
languages=['en'],
dev=False,
force=False)
filename = self._filename('genfiles', 'javascript', 'en',
'video-package-*.js')
self.assertIn('readable_js_packages_prod',
os.readlink(ka_root.join(filename)))
self._build(['js'],
readable=False,
languages=['en'],
dev=False,
force=False)
filename = self._filename('genfiles', 'javascript', 'en',
'video-package-*.js')
self.assertIn('compressed_js_packages_prod',
os.readlink(ka_root.join(filename)))
# All deps are already built, but this should still change symlinks.
self._build(['js'],
readable=True,
languages=['en'],
dev=False,
force=False)
filename = self._filename('genfiles', 'javascript', 'en',
'video-package-*.js')
self.assertIn('readable_js_packages_prod',
os.readlink(ka_root.join(filename)))
def lint_have_needed_babel_locales(files_to_lint):
"""Make sure we have all the locales we need, in third_party/babel.
third_party/babel/localedata comes with 664 languages, which is
great for coverage but bad for deploy time.
So to speed things up, I added to app.yaml's skip_files all
language files that aren't used by either a locale in all_ka_locales or
a YouTube locale.
This lint check makes sure that when we update those lists (or update the
babel subrepo), we upload any localedata languages that we need to.
"""
if (ka_root.join('intl', 'i18n.py') not in files_to_lint and
not any(f.startswith(intl.data.INTL_VIDEO_PLAYLISTS_DIR)
for f in files_to_lint) and
ka_root.join('third_party', 'babel-khansrc') not in files_to_lint):
return
config = modules_util.module_yaml('default')
# Take only the rules for third_party/babel/localedata, and strip
# off that prefix since we're starting the FileIterator in the
# localedata directory rather than ka-root.
# Note this depends on the babel rules starting with ^ and ending with $.
localedata_root = 'third_party/babel/localedata'
prefix = re.escape(r'(?:.*/webapp/)?')
babel_regexps = [s for s in re.findall(r'\^(?:%s)?([^$]*)\$' % prefix,
config['skip_files'].regex.pattern)
if s.startswith(localedata_root + '/')]
skip_files = [s.replace('%s/' % localedata_root, '^')
for s in babel_regexps]
skip_re = re.compile('|'.join('(?:%s)' % p for p in skip_files))
orig_level = logging.getLogger().level
try:
logging.getLogger().setLevel(logging.ERROR)
localedata_files = appcfg.FileIterator(ka_root.join(localedata_root),
skip_re, config['runtime'])
localedata_files = list(localedata_files)
finally:
logging.getLogger().setLevel(orig_level)
# Remove the '.dat' extension.
all_locales_for_babel = frozenset(os.path.splitext(f)[0]
for f in localedata_files)
needed_locales = intl.data.all_ka_locales(include_english=True)
babel_locales = set([b for b in [intl.locale.ka_locale_to_babel(l)
for l in needed_locales] if b])
for babel_locale in babel_locales:
# We need to check zh_Hans_CN.dat exists, but also zh_Hans.dat, etc.
for prefix in intl.locale.locale_prefixes(babel_locale):
# We need to convert from KA-style - to babel-style _.
prefix = prefix.replace('-', '_')
if prefix not in all_locales_for_babel:
yield ('skip_files.yaml', 1,
"We need babel locale info for %s but it's been added"
" to skip-files (need to whitelist it)." % prefix)
def test_javascript_manifest_toc(self):
self._build(['js_and_css'],
readable=True,
languages=['en'],
dev=False,
force=False,
gae_version='161616-2233-hello')
toc_filename = ka_root.join('genfiles', 'manifests',
'toc-161616-2233-hello.json')
with open(toc_filename) as f:
toc = json.load(f)
self.assertIn('en', toc)
self.assertNotIn('fakelang', toc)
en_file = ka_root.join('genfiles', 'manifests', 'en',
'package-manifest-%s.js' % toc['en'])
self.assertFileExists(en_file)
self._build(['js_and_css'],
readable=True,
languages=['fakelang'],
dev=False,
force=False,
gae_version='161616-2233-hello')
with open(toc_filename) as f:
toc = json.load(f)
self.assertIn('en', toc)
self.assertIn('fakelang', toc)
self.assertFileExists(en_file)
fake_file = ka_root.join('genfiles', 'manifests', 'fakelang',
'package-manifest-%s.js' % toc['fakelang'])
self.assertFileExists(fake_file)
def _lint(self, contents_map):
for (fname, imports) in contents_map.iteritems():
import_lines = []
for i in imports:
if i.startswith(' '): # indicates a late import
import_lines.append('def foo():\n import %s' % i.strip())
else:
import_lines.append('import %s' % i)
self.set_file_contents(ka_root.join(fname),
'\n'.join(import_lines))
return import_lint._lint_unnecessary_late_imports(
[ka_root.join(fname) for fname in contents_map])
def test_python_manifest(self):
self._build(['js'], readable=True, languages=['en'], dev=False)
fname = ('genfiles/readable_manifests_prod/en/'
'javascript-md5-packages.json')
with open(ka_root.join(fname)) as f:
manifest = json.load(f)
self.assertIn('shared.js', manifest)
self._build(['css'], readable=True, languages=['en'], dev=False)
fname = ('genfiles/readable_manifests_prod/en/'
'stylesheets-md5-packages.json')
with open(ka_root.join(fname)) as f:
manifest = json.load(f)
self.assertIn('video.css', manifest)
def test_javascript_and_python_manifest_symlinks(self):
self._build(['js_and_css'],
readable=True,
languages=['en', 'fakelang'],
dev=False)
f1 = 'genfiles/readable_manifests_prod/en/package-manifest.js'
f2 = self._filename('genfiles', 'manifests', 'en',
'package-manifest-*.js')
self.assertEqual(os.path.realpath(ka_root.join(f1)),
os.path.realpath(ka_root.join(f2)))
f3 = 'genfiles/readable_manifests_prod/fakelang/package-manifest.js'
f4 = self._filename('genfiles', 'manifests', 'fakelang',
'package-manifest-*.js')
self.assertEqual(os.path.realpath(ka_root.join(f3)),
os.path.realpath(ka_root.join(f4)))
# The python manifests should have the same md5sum as the
# javascript manifests.
for js_or_css in ('javascript', 'stylesheets'):
f1b = f1.replace('package-manifest.js',
'%s-md5-packages.json' % js_or_css)
f2b = f2.replace('package-manifest-',
'%s-md5-packages-' % js_or_css) + 'on' # js->json
f3b = f3.replace('package-manifest.js',
'%s-md5-packages.json' % js_or_css)
f4b = f4.replace('package-manifest-',
'%s-md5-packages-' % js_or_css) + 'on' # js->json
self.assertFileExists(f1b)
self.assertFileExists(f2b)
self.assertFileExists(f3b)
self.assertFileExists(f4b)
self.assertEqual(os.path.realpath(ka_root.join(f1b)),
os.path.realpath(ka_root.join(f2b)))
self.assertEqual(os.path.realpath(ka_root.join(f3b)),
os.path.realpath(ka_root.join(f4b)))
def test_compress(self):
# Our mocked-out 'compressor' just gets rid of newlines
self._build(['shared.js', 'video.css'],
readable=False,
languages=['en'],
dev=True,
force=False)
self.assertFileContentsMatch(
ka_root.join('genfiles', 'compressed_javascript', 'en', 'genfiles',
'compiled_es6', 'en', 'javascript', 'shared-package',
'dev.min.js'), 'expected/test_compress/dev.min.js')
self.assertFileContentsMatch(
ka_root.join('genfiles', 'compressed_stylesheets', 'en',
'stylesheets', 'video-package', 'video.less.min.css'),
'expected/test_compress/video.less.min.css')
def _lint_single_wsgi_entrypoint_import(filename):
"""Returns a lint-error tuple, or None if the file is ok."""
with open(ka_root.join(filename)) as f:
module_ast = ast.parse(f.read())
for stmt in module_ast.body:
# If we see a string first, that's the docstring; that's ok, we'll
# check the next one.
if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Str):
continue
# We also allow __future__ imports first, because python wants it
# that way. Those have to be in 'from __future__ import *' format.
elif (isinstance(stmt, ast.ImportFrom) and
stmt.module == '__future__'):
continue
# If the first import is appengine_config, we're happy! Since
# appengine_config is toplevel, we don't have to worry about
# from-imports.
elif (isinstance(stmt, ast.Import) and
stmt.names[0].name == 'appengine_config'):
return None
# Otherwise, we're sad. Return a lint error.
else:
return (filename, stmt.lineno,
"Must import appengine_config before any other "
"(non-__future__) imports.")
# If the file has nothing other than docstrings and __future__ imports,
# something has gone horribly wrong; consider it an error to be safe.
return (filename, 1,
"This file doesn't look like a WSGI entrypoint! Are you sure "
"handlers-*.yaml is set up right?")
def lint_every_rendered_component_has_a_fixture(files_to_lint):
"""Check that every component we render has an associated fixture file.
In order to test that a particular react component can be
server-side rendered, we need to actually try to render it with a
particular value for props. This is what component.fixture.js
files are for. We just make sure the people write them!
For now, we allow the fixture file to be empty (just `[]`). Later
we may insist on actually useful fixtures.
"""
files_to_lint = lintutil.filter(files_to_lint, suffix='.html')
for f in files_to_lint:
contents_of_f = lintutil.file_contents(f)
for m in RENDER_REACT_RE.finditer(contents_of_f):
component_file = m.group(1)
# To be server-side renderable, the fixture file has to be
# a javascript file, not jsx or something else.
fixture_file = component_file + '.fixture.js'
if not os.path.exists(ka_root.join(fixture_file)):
linenum = contents_of_f.count('\n', 0, m.start()) + 1
yield (f, linenum,
'%s must have an associated fixture file %s' %
(component_file, fixture_file))
def lint_every_rendered_component_has_a_fixture(files_to_lint):
"""Check that every component we render has an associated fixture file.
In order to test that a particular react component can be
server-side rendered, we need to actually try to render it with a
particular value for props. This is what component.fixture.js
files are for. We just make sure the people write them!
For now, we allow the fixture file to be empty (just `[]`). Later
we may insist on actually useful fixtures.
"""
files_to_lint = lintutil.filter(files_to_lint, suffix='.html')
for f in files_to_lint:
contents_of_f = lintutil.file_contents(f)
for m in RENDER_REACT_RE.finditer(contents_of_f):
component_file = m.group(1)
# To be server-side renderable, the fixture file has to be
# a javascript file, not jsx or something else.
fixture_file = component_file + '.fixture.js'
if not os.path.exists(ka_root.join(fixture_file)):
linenum = contents_of_f.count('\n', 0, m.start()) + 1
yield (f, linenum,
'%s must have an associated fixture file %s'
% (component_file, fixture_file))
def test_javascript_manifest_dev(self):
self._build(['js_and_css'], readable=True, languages=['en'], dev=True)
fname = 'genfiles/readable_manifests_dev/en/package-manifest.js'
with open(ka_root.join(fname)) as f:
contents = f.read().strip()
# The json is inside this javascript.
manifest = contents.split('{', 1)[1]
manifest = manifest.rsplit('}', 1)[0]
manifest = json.loads('{' + manifest + '}')
self.assertEqual({'javascript', 'stylesheets'}, set(manifest.keys()))
shared = [
e for e in manifest['javascript'] if e['name'] == 'shared.js'
]
self.assertEqual(1, len(shared), shared)
self.assertEqual(('/_kake/genfiles/readable_js_packages_dev/en/'
'shared-package.js'), shared[0]['url'])
# shared.js has no dependencies.
self.assertNotIn('dependencies', shared[0])
video = [
e for e in manifest['stylesheets'] if e['name'] == 'video.css'
]
self.assertEqual(1, len(video), video)
self.assertEqual(('/_kake/genfiles/readable_css_packages_dev/en/'
'video-package.css'), video[0]['url'])
self.assertEqual(['tiny.css'], video[0]['dependencies'])
def filter(files_to_lint, prefix='', suffix='', exclude_substrings=[]):
"""Return a filtered version of files to lint matching prefix AND suffix.
First it converts each file in files_to_lint to a relative
filename (relative to ka_root). Then it makes sure
relpath.startswith(prefix) and relpath.endswith(suffix).
exclude_substrings is a list: all files which include any
substring in that list is excluded. For exclude_substrings,
the full abspath of the file is considered.
It then converts matching files back to an abspath and returns them.
prefix and suffix can be the same as for startswith and endswith:
either a single string, or a list of strings which are OR-ed
together.
"""
without_excludes = [
f for f in files_to_lint if not any(s in f for s in exclude_substrings)
]
relpaths = [ka_root.relpath(f) for f in without_excludes]
filtered = [
f for f in relpaths if f.startswith(prefix) and f.endswith(suffix)
]
filtered_abspaths = [ka_root.join(f) for f in filtered]
return filtered_abspaths
def included_handlebars_files(self, handlebars_infile, context):
"""Return a list of filepaths the given template templates on."""
# TODO(jlfwong): This loses out on the caching made available in
# ComputedIncludeInputs.included_files.
deps = [
'third_party/javascript-khansrc/'
'handlebars/handlebars.runtime.js'
]
# A handlebars include of another handlebars file is never at
# the 'top level', since {{>...}} and {{#invokePartial ...}}
# always resolve to functions.
if self.top_level_only:
return deps
with open(ka_root.join(handlebars_infile)) as f:
contents = f.read()
for pattern in [self._INVOKE_PARTIAL_RE, self._PARTIAL_RE]:
for m in pattern.finditer(contents):
package_name = m.group(1)
template_name = m.group(2)
deps.append('javascript/%s-package/%s.handlebars' %
(package_name, template_name))
return deps
def _lint_single_wsgi_entrypoint_import(filename):
"""Returns a lint-error tuple, or None if the file is ok."""
with open(ka_root.join(filename)) as f:
module_ast = ast.parse(f.read())
for stmt in module_ast.body:
# If we see a string first, that's the docstring; that's ok, we'll
# check the next one.
if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Str):
continue
# We also allow __future__ imports first, because python wants it
# that way. Those have to be in 'from __future__ import *' format.
elif (isinstance(stmt, ast.ImportFrom)
and stmt.module == '__future__'):
continue
# If the first import is appengine_config, we're happy! Since
# appengine_config is toplevel, we don't have to worry about
# from-imports.
elif (isinstance(stmt, ast.Import)
and stmt.names[0].name == 'appengine_config'):
return None
# Otherwise, we're sad. Return a lint error.
else:
return (filename, stmt.lineno,
"Must import appengine_config before any other "
"(non-__future__) imports.")
# If the file has nothing other than docstrings and __future__ imports,
# something has gone horribly wrong; consider it an error to be safe.
return (filename, 1,
"This file doesn't look like a WSGI entrypoint! Are you sure "
"handlers-*.yaml is set up right?")
def test_javascript_manifest_prod(self):
self._build(['js_and_css'],
readable=False,
languages=['en'],
dev=False)
fname = 'genfiles/compressed_manifests_prod/en/package-manifest.js'
with open(ka_root.join(fname)) as f:
contents = f.read().strip()
# The json is inside this javascript.
manifest = contents.split('{', 1)[1]
manifest = manifest.rsplit('}', 1)[0]
manifest = json.loads('{' + manifest + '}')
self.assertEqual({'javascript', 'stylesheets'}, set(manifest.keys()))
shared = [
e for e in manifest['javascript'] if e['name'] == 'shared.js'
]
self.assertEqual(1, len(shared), shared)
self.assertTrue(
shared[0]['url'].startswith(
'/genfiles/javascript/en/shared-package-'), shared[0])
self.assertFileExists(shared[0]['url'][1:])
video = [
e for e in manifest['stylesheets'] if e['name'] == 'video.css'
]
self.assertEqual(1, len(video), video)
self.assertTrue(
video[0]['url'].startswith(
'/genfiles/stylesheets/en/video-package-'), video[0])
self.assertFileExists(video[0]['url'][1:])
def test_simple(self):
build_prod_main.main(['js'], ['en'], {}, False, True, True)
outdir = self._abspath('genfiles', 'readable_js_packages_prod', 'en')
self.assertEqual([
'corelibs-package.js',
'corelibs-package.js.deps',
'corelibs-package.js.map',
'shared-package.js',
'shared-package.js.deps',
'shared-package.js.map',
'third-party-package.js',
'third-party-package.js.deps',
'third-party-package.js.map',
'video-package.js',
'video-package.js.deps',
'video-package.js.map',
], sorted(os.listdir(outdir)))
# These will die if the symlinks don't exist.
symlinks = (
self._filename('genfiles', 'javascript', 'en',
'corelibs-package-*.js'),
self._filename('genfiles', 'javascript', 'en',
'shared-package-*.js'),
self._filename('genfiles', 'javascript', 'en',
'third-party-package-*.js'),
self._filename('genfiles', 'javascript', 'en',
'video-package-*.js'),
)
for symlink in symlinks:
self.assertIn('readable_js_packages_prod',
os.readlink(ka_root.join(symlink)))
def _lint(self, python_contents):
"""Superclass's assert_error() and assert_no_error call this."""
path = ka_root.join('using_gettext_at_import_time_file.py')
self.set_file_contents(path, python_contents)
# We also actually have to write the file to make it importable.
with open(os.path.join(self.tmpdir, os.path.basename(path)), 'w') as f:
print >> f, python_contents
with mock.patch('shared.ka_root.root', self.tmpdir):
return i18n_lint.lint_not_using_gettext_at_import_time([path])
def test_depends_on_image_file(self):
self._build(['video.css'],
readable=False,
languages=['en'],
dev=True,
force=False)
# Now replace tiny.png with other.png
shutil.copy(ka_root.join('images', 'other.png'),
ka_root.join('images', 'tiny.png'))
filemod_db.clear_mtime_cache() # since we modified a file
# Now we should rebuild because of the changed image file.
self._build(['video.css'],
readable=False,
languages=['en'],
dev=True,
force=False)
outfile = 'genfiles/compressed_css_packages_dev/en/video-package.css'
self.assertFileContains(outfile, self.other_png_base64)
def _graphie_label_files():
"""Yield all label files that should be translated.
Returned filenames are relative to ka-root.
We assume that all of the files in genfiles/labels are to-translate.
"""
with open(ka_root.join('intl', 'translations',
'graphie_image_shas.json')) as f:
for sha in json.load(f).keys():
yield os.path.join('genfiles', 'labels', 'en',
'%s-data.json' % sha)
with open(
ka_root.join('intl', 'translations',
'graphie_image_shas_in_articles.json')) as f:
for sha in json.load(f).keys():
yield os.path.join('genfiles', 'labels', 'en',
'%s-data.json' % sha)
def test_no_symlinks_for_dev(self):
self._build(['js_and_css'],
readable=True,
languages=['en'],
dev=True,
force=False)
for path in ('javascript/*', 'stylesheets/*',
'genfiles/*/package-manifest.toc.json',
'package-manifest-*'):
self.assert_glob_matches_zero(ka_root.join('genfiles', path))
def test_delete_obsolete_files_many_languages(self):
self._build(['js'],
readable=False,
languages=['en', 'fakelang'],
dev=False,
force=False)
_write_to_file(ka_root.join('javascript', 'video-package', 'video.js'),
'var Video = {\n youtubeId: "all new!"\n };')
_write_to_file(
ka_root.join('genfiles', 'translations', 'fakelang', 'javascript',
'video-package', 'video.js'),
'var Video = {youtubeId: "fake new!"};')
filemod_db.clear_mtime_cache() # since we modified a file
# We just test that this doesn't raise a FileNotFound error.
self._build(['js'],
readable=False,
languages=['en', 'fakelang'],
dev=False,
force=False)
def test_creates_only_necessary_language_entries(self):
self._build(['js'],
readable=False,
languages=['en', 'fakelang'],
dev=False,
force=False)
# Make sure only the 'en' file is in the manifest for shared.js.
with open(
ka_root.join('genfiles', 'compressed_manifests_prod', 'en',
'javascript-md5-packages.json')) as f:
en_manifest = json.load(f)
with open(
ka_root.join('genfiles', 'compressed_manifests_prod',
'fakelang', 'javascript-md5-packages.json')) as f:
fakelang_manifest = json.load(f)
self.assertEqual(en_manifest['shared.js'],
fakelang_manifest['shared.js'])
self.assertNotIn("genfiles/javascript/fakelang",
fakelang_manifest['shared.js'])
def test_simple_import(self):
actual = self._imports("import json\nimport api.internal")
self.assertEqual(actual[0].filename, ka_root.join("content/test.py"))
self.assertEqual(actual[0].lineno, 1)
self.assertEqual(actual[0].is_toplevel, True)
self.assertEqual(actual[0].module, "json")
self.assertEqual(actual[0].name, "json")
self.assertEqual(actual[0].level, 0)
self.assertEqual(actual[0].has_comma, False)
self.assertFalse(actual[0].has_from, False)
self.assertFalse(actual[0].has_as, False)
self.assertEqual(actual[1].filename, ka_root.join("content/test.py"))
self.assertEqual(actual[1].lineno, 2)
self.assertEqual(actual[1].is_toplevel, True)
self.assertEqual(actual[1].module, "api.internal")
self.assertEqual(actual[1].name, "api.internal")
self.assertEqual(actual[0].level, 0)
self.assertEqual(actual[0].has_comma, False)
self.assertFalse(actual[0].has_from, False)
self.assertFalse(actual[0].has_as, False)
def run_eslint(js_filename_contents_pairs):
"""Run eslint on the given content, given as (filename, content) pairs.
The output is an array of (<filename>, <linenum>, <error message>)
triples.
This re-uses the eslint params that khan-linter uses. We prefer
the khan-linter version in devtools, but use the one in
third-party if needed.
"""
eslintrc = os.path.join(ka_root.root, '..',
'devtools', 'khan-linter', 'eslintrc.browser')
if not os.path.exists(eslintrc):
eslintrc = ka_root.join('third_party', 'khan-linter-src',
'eslintrc.browser')
# Our eslint runner expects a file that looks like:
# <eslint filename>
# -- LINT_JAVASCRIPT_IN_HTML SEPARATOR: filename\n
# <file contents>
# -- LINT_JAVASCRIPT_IN_HTML SEPARATOR: filename\n
# ...
eslint_input = [eslintrc + '\n']
for (f, contents) in js_filename_contents_pairs:
eslint_input.append('-- LINT_JAVASCRIPT_IN_HTML SEPARATOR: %s\n' % f)
# Make sure when we join the strings together, each is on its own line.
if not contents.endswith('\n'):
contents += '\n'
eslint_input.append(contents)
eslint_runner = ka_root.join('testutil', 'lint_javascript_in_html.js')
p = subprocess.Popen(['node', eslint_runner],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate(''.join(eslint_input))
if stderr:
raise RuntimeError("Unexpected stderr from eslint:\n%s" % stderr)
return [tuple(l.split(':', 2)) for l in stdout.splitlines()]
def _is_a_first_party_package(module_name):
"""Return true if module_name is a first-party package (aka dir).
A package is a directory, as opposed to a module which is a filename.
We say whether module_name, when interpreted relative to ka_root is
a package or not. It only is if
ka_root/module/converted/to/path/__init__.py
exists.
"""
if module_name not in _first_party_package_cache:
package = ka_root.join(*(module_name.split('.') + ['__init__.py']))
_first_party_package_cache[module_name] = os.path.exists(package)
return _first_party_package_cache[module_name]
def _image_urls_and_file_info(content):
"""Given an image-url string, return an iterator with image info."""
matches = _CSS_IMAGE_RE.finditer(content)
for m in matches:
relative_filename = m.group(1)[1:] # relative to ka-root
# Sometimes urls have ?'s (url queries) in them to bust
# caches. Those are not part of the filename. :-)
relative_filename = relative_filename.split('?')[0]
pathname = ka_root.join(relative_filename)
try:
filesize = os.stat(pathname).st_size
yield (m.group(1), relative_filename, filesize)
except OSError: # file not found
log.warning('reference to non-existent image %s', pathname)
def _jinja2_files():
"""Yield all jinja2 files that might have text-to-translate.
Returned filenames are relative to ka-root.
We assume all .html and .txt files under templates/ is jinja2.
"""
root = ka_root.join('templates')
for (rootdir, dirs, files) in os.walk(root):
reldir = ka_root.relpath(rootdir)
for f in files:
if f.endswith('.html') or f.endswith('.txt'):
relpath = os.path.join(reldir, f)
yield os.path.normpath(relpath)
def _update_image_url_info(css_filename, image_url_info):
"""Given css_filenames relative to ka-root, update _IMAGE_URL_INFO.
Returns:
A list of image filenames, relative to ka-root, mentioned in
this css-filename.
"""
# First, we need to delete all old references to css_filenames.
for file_info in image_url_info.itervalues():
new_files = [f for f in file_info[0] if f != css_filename]
if len(new_files) < len(file_info[0]):
# We go through this contortion so we can edit the list in place.
del file_info[0][:]
file_info[0].extend(new_files)
# If the file no longer exists (has been deleted), we're done!
if not os.path.exists(ka_root.join(css_filename)):
log.v3("removing image-url info for %s: it's been deleted",
css_filename)
return
# Then, we need to add updated references, based on the current
# file contents.
log.v2('Parsing image-urls from %s', css_filename)
with open(ka_root.join(css_filename)) as f:
content = f.read()
retval = []
for (img_url, img_relpath,
img_size) in (_image_urls_and_file_info(content)):
image_url_info.setdefault(img_url, ([], img_relpath, img_size))
image_url_info[img_url][0].append(css_filename)
retval.append(img_relpath)
log.v4('Image-url info: %s', retval)
return retval
def test_combine_only(self):
self._build(['js'],
readable=True,
languages=['en'],
dev=True,
force=False)
outfile = ka_root.join('genfiles', 'readable_js_packages_dev', 'en',
'video-package.js')
self.assertFileContains(outfile,
'var Video = {\n youtubeId: "en"\n };')
self.assertFileContains(
outfile, 'var ModalVideo = {t: require("./t.handlebars"); };')
# A string taken from the compiled handlebars template.
self.assertFileContains(outfile,
'var template = Handlebars.template(function')
def test_language(self):
"""Ensure nothing bad happens to lang1 when we build lang2."""
self.write_fakelang_pofile()
build_prod_main.main(['js'], ['en'], readable=True)
symlinks = (
self._filename('genfiles', 'javascript', 'en',
'corelibs-package-*.js'),
self._filename('genfiles', 'javascript', 'en',
'shared-package-*.js'),
self._filename('genfiles', 'javascript', 'en',
'third-party-package-*.js'),
self._filename('genfiles', 'javascript', 'en',
'video-package-*.js'),
)
for symlink in symlinks:
self.assertIn('readable_js_packages_prod',
os.readlink(ka_root.join(symlink)))
build_prod_main.main(['js'], ['fakelang'], readable=True)
# Now check the 'en' files again -- they shouldn't have been
# deleted just because we built fakelang.
for symlink in symlinks:
self.assertIn('readable_js_packages_prod',
os.readlink(ka_root.join(symlink)))
def _handlebars_files():
"""Yield all handlebars files that might have text-to-translate.
Returned filenames are relative to ka-root.
We assume all .handlebars under javascript needs to be translated. We
have an intl.english_only.should_not_translate check later in this file to
filter out the ones we don't care about.
"""
root = ka_root.join('javascript')
for (rootdir, dirs, files) in os.walk(root):
reldir = ka_root.relpath(rootdir)
for f in files:
if f.endswith('.handlebars'):
relpath = os.path.join(reldir, f)
yield os.path.normpath(relpath)
def setUp(self):
super(TestBase, self).setUp()
self._copy_to_test_tmpdir(
os.path.join('kake', 'compile_js_css_packages-testfiles'))
# We need the kake directory to read compile_handlebars.js, etc.
# NOTE: We intentionally make a copy instead of symlinking here because
# node resolves dependencies based on the where the real file is. We
# want compile_handlebars.js to look inside the sandbox for
# dependencies, not the real ka-root.
shutil.copytree(os.path.join(self.real_ka_root, 'kake'),
self._abspath('kake'))
os.makedirs(ka_root.join('js_css_packages'))
# We copy here because filemod-db doesn't like out-of-tree symlinks.
f = 'js_css_packages/third_party_js.py'
shutil.copyfile(os.path.join(self.real_ka_root, f), self._abspath(f))
self.mock_value('intl.data._LanguageStatus._DID_LANGUAGE_CHECKS', True)
def lint_symbol_imports(files_to_lint):
"""Find cases where we import a symbol rather than a module.
For instance, "from main import application". That is not a module,
it's a variable defined within a module.
"""
files_to_lint = lintutil.filter(files_to_lint, suffix='.py')
for f in files_to_lint:
import_lines = _get_import_lines(f)
for import_line in import_lines:
# You can only import a symbol via the 'from ... import' syntax
if not import_line.has_from:
continue
if import_line.module.startswith('__future__'): # not a real dir
continue
# TODO(csilvers): our style guide has an exception
# allowing the import of individual symbols from
# third-party packages which document this as a best
# practice. Add a special-case to ignore those, here.
# (Or amend the style guide to remove the exception :-) )
if not _is_a_module_or_package(import_line.module):
# While we may not be a module or a package, the thing
# containing us should definitely be. If it's not, or
# if it's a package but an empty one, then we're
# importing something that doesn't (now) exist,
# probably an auto-generated package. We just ignore
# it; we can't say anything useful about it.
module_parent = import_line.module.rsplit('.', 1)[0]
module_parent_as_package = ka_root.join(
module_parent.replace('.', os.sep), '__init__.py')
if (not _is_a_module_or_package(module_parent) or
(os.path.exists(module_parent_as_package)
and os.stat(module_parent_as_package).st_size == 0)):
continue
actual_module = import_line.module.rsplit('.', 1)[0]
yield (f, import_line.lineno,
"Import the whole module %s, not individual symbols "
"inside it" % actual_module)
def _get_top_level_imports(module):
if module not in _direct_top_level_imports:
# We only care about first-party files here, so we assume
# that this module is relative to ka-root. If not, then
# it's safe to ignore it anyway! We also ignore third-party
# files explicitly.
if module.startswith('third_party.'):
path = '' # force a file-ignore
else:
path = module.replace('.', os.sep) + '.py'
import_lines = _get_import_lines(ka_root.join(path))
if import_lines is None: # file not found
_direct_top_level_imports[module] = None
_direct_late_imports[module] = None
else:
_direct_top_level_imports[module] = frozenset(
[il.module for il in import_lines if il.is_toplevel])
_direct_late_imports[module] = frozenset(
[il.module for il in import_lines if not il.is_toplevel])
return _direct_top_level_imports[module] or set()
def filter(files_to_lint, prefix='', suffix='', exclude_substrings=[]):
"""Return a filtered version of files to lint matching prefix AND suffix.
First it converts each file in files_to_lint to a relative
filename (relative to ka_root). Then it makes sure
relpath.startswith(prefix) and relpath.endswith(suffix).
exclude_substrings is a list: all files which include any
substring in that list is excluded. For exclude_substrings,
the full abspath of the file is considered.
It then converts matching files back to an abspath and returns them.
prefix and suffix can be the same as for startswith and endswith:
either a single string, or a list of strings which are OR-ed
together.
"""
without_excludes = [f for f in files_to_lint
if not any(s in f for s in exclude_substrings)]
relpaths = [ka_root.relpath(f) for f in without_excludes]
filtered = [f for f in relpaths
if f.startswith(prefix) and f.endswith(suffix)]
filtered_abspaths = [ka_root.join(f) for f in filtered]
return filtered_abspaths
def lint_missing_used_context_keys(files_to_lint):
"""Attempts to find places the user failed to update used_context_keys().
If you write a compile_rule that uses context['foo'], you're
supposed to advertise that fact by including 'foo' in your
used_context_keys() method. But it's easy to forget to do that.
This rule attempts to remind you by looking at the source code for
your class and trying to find all uses.
This isn't perfect, which is why it's a lint rule and we don't
just automatically extract uses of context['foo'], but it's better
than nothing! If it's claiming a line is a use of context when
it's not, just stick a @Nolint at the end of the line.
"""
# Only files under the kake directory might have compile rules.
relfiles_to_lint = [ka_root.relpath(f) for f in files_to_lint
if ka_root.relpath(f).startswith('kake/')]
if not relfiles_to_lint:
return
# This forces us to import all the kake compile_rules.
from kake import make # @UnusedImport
classes = (list(_all_subclasses(compile_rule.CompileBase)) +
list(_all_subclasses(computed_inputs.ComputedInputsBase)))
for cls in classes:
class_file = cls.__module__.replace('.', '/') + '.py'
if class_file not in relfiles_to_lint:
continue
claimed_used_context_keys = set(cls.used_context_keys())
actual_used_context_keys = {} # map from key to linenum where used
class_source = inspect.getsource(cls)
module_source = inspect.getsource(sys.modules[cls.__module__])
# Find what line-number the class we're linting starts on.
class_source_pos = module_source.find(class_source)
class_startline = module_source.count('\n', 0, class_source_pos) + 1
# Find what line-number class.used_context_keys() starts on.
used_context_keys_pos = class_source.find('def used_context_keys')
if used_context_keys_pos == -1:
used_context_keys_line = 1
else:
used_context_keys_line = (class_source.count('\n', 0,
used_context_keys_pos)
+ class_startline)
for m in _CONTEXT_USE_RE.finditer(class_source):
if '@Nolint' not in m.group(0):
key = m.group(1) or m.group(2) or m.group(3)
linenum = (class_source.count('\n', 0, m.start())
+ class_startline)
actual_used_context_keys.setdefault(key, linenum)
must_add = set(actual_used_context_keys) - claimed_used_context_keys
must_remove = claimed_used_context_keys - set(actual_used_context_keys)
for key in must_add:
# We don't require people to register system keys (start with _)
# or glob vars (start with '{').
if not key.startswith(('_', '{')):
yield (ka_root.join(class_file),
actual_used_context_keys[key], # linenum
'Build rule uses "%s" but it is not listed in'
' used_context_keys(). Add it there or mark'
' it with @Nolint if this is in error.' % key)
for key in must_remove:
yield (ka_root.join(class_file),
used_context_keys_line,
'Build rule does not use "%s" but it is listed in'
' used_context_keys(). Add it there or fix this'
' linter if it is in error.' % key)
def lint_not_using_gettext_at_import_time(files_to_lint):
"""Make sure we don't use i18n._/etc in a static context.
If you have a global variable such as '_FOO = i18n._("bar")', at
the top of some .py file, it won't work the way you intend because
i18n._() needs to be called while handling a request in order to
know what language to translate to. (Instead, you'd need to do
_FOO = lambda: i18n._("bar")
or some such.)
This tests for this by mocking i18n._ et al., and then importing
everything (but running nothing). Any i18n._ calls that happen
during this import are problematic! We have to spawn a new
python process to make sure we do the importing properly (and
without messing with the currently running python environment!)
"""
candidate_files_to_lint = lintutil.filter(files_to_lint, suffix='.py')
files_to_lint = []
for filename in candidate_files_to_lint:
contents = lintutil.file_contents(filename)
# Check that it's plausible this file uses i18n._ or similar.
# This also avoids importing random third-party files that may
# have nasty side-effects at import time (all our code is too
# well-written to do that!)
if 'import intl' in contents or 'from intl' in contents:
files_to_lint.append(filename)
program = """\
import os # @Nolint(linter can't tell this is in a string!)
import sys # @Nolint(linter can't tell this is in a string!)
import traceback
import intl.request # @Nolint(seems unused to our linter but it's used)
_ROOT = "%s"
def add_lint_error(f):
# We assume code in 'intl' doesn't make this mistake, and thus
# the first stack-frame before we get into 'intl' is the
# offending code. ctx == '<string>' means the error occurred in
# this pseudo-script.
for (ctx, lineno, fn, line) in reversed(traceback.extract_stack()):
if os.path.isabs(ctx):
ctx = os.path.relpath(ctx, _ROOT)
if ctx != '<string>' and not ctx.startswith('intl/'):
if ctx == f:
print 'GETTEXT ERROR {} {}'.format(ctx, lineno)
break
return 'en' # a fake value for intl.request.ka_locale
""" % ka_root.root
if not files_to_lint:
return
for filename in files_to_lint:
modulename = ka_root.relpath(filename)
modulename = os.path.splitext(modulename)[0] # nix .py
modulename = modulename.replace('/', '.')
# Force a re-import.
program += 'sys.modules.pop("%s", None)\n' % modulename
program += ('intl.request.ka_locale = lambda: add_lint_error("%s")\n'
% ka_root.relpath(filename))
program += 'import %s\n' % modulename
p = subprocess.Popen(
['env', 'PYTHONPATH=%s' % ':'.join(sys.path),
sys.executable, '-c', program],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
p.wait()
lint_output = p.stdout.read()
for line in lint_output.splitlines():
if line.startswith('GETTEXT ERROR '):
line = line[len('GETTEXT ERROR '):]
(filename, linenum) = line.rsplit(' ', 1)
yield (ka_root.join(filename), int(linenum),
'Trying to translate at import-time, but '
'translation only works at runtime! '
'Use intl.i18n.mark_for_translation() instead.')
def _lint(self, contents):
fname = ka_root.join("content", "test.py")
self.set_file_contents(fname, contents)
return import_lint.lint_symbol_imports([fname])
def _lint(self, contents):
self.set_file_contents(ka_root.join('test.py'), contents)
return import_lint.lint_no_backslashes([ka_root.join('test.py')])
def _lint(self, contents):
self.set_file_contents(ka_root.join('test.py'), contents)
return import_lint.lint_absolute_import([ka_root.join('test.py')])
def setUp(self):
super(PackageImportTest, self).setUp()
files = {
ka_root.join('content/__init__.py'),
ka_root.join('content/frozen_model.py'),
ka_root.join('content/videos/__init__.py'),
ka_root.join('content/videos/frozen_video.py'),
ka_root.join('content/articles/__init__.py'),
ka_root.join('content/articles/frozen_article.py'),
ka_root.join('content/articles/testdata/__init__.py'),
ka_root.join('content/articles/testdata/mytest.py'),
ka_root.join('emails/__init__.py'),
ka_root.join('emails/send.py'),
ka_root.join('emails/templates/__init__.py'),
ka_root.join('third_party/boto/__init__.py'),
ka_root.join('third_party/boto/auth.py'),
ka_root.join('third_party/vendored/third_party/idna/__init__.py'),
ka_root.join('third_party/vendored/third_party/idna/codec.py'),
ka_root.join('import_lint.py'),
'/usr/lib/python/lxml/__init__.py',
'/usr/lib/python/lxml/parse.py',
}
patcher = mock.patch('os.path.exists', lambda f: f in files)
self.addCleanup(patcher.stop)
patcher.start()
def _imports(self, content):
self.set_file_contents(ka_root.join('content/test.py'), content)
return import_lint._get_import_lines(ka_root.join('content/test.py'))
def _lint(self, contents):
self.set_file_contents(ka_root.join('test.py'), contents)
return import_lint.lint_redundant_imports([ka_root.join('test.py')])
def setUp(self):
super(SymbolImportTest, self).setUp()
files = {
ka_root.join('content/__init__.py'),
ka_root.join('content/cool_module.so'),
ka_root.join('content/frozen_model.py'),
ka_root.join('content/videos/__init__.py'),
ka_root.join('content/videos/frozen_video.py'),
ka_root.join('content/articles/__init__.py'),
ka_root.join('content/articles/frozen_article.py'),
ka_root.join('content/articles/testdata/__init__.py'),
ka_root.join('content/articles/testdata/mytest.py'),
ka_root.join('emails/__init__.py'),
ka_root.join('emails/send.py'),
ka_root.join('emails/templates/__init__.py'),
ka_root.join('third_party/__init__.py'),
ka_root.join('third_party/boto/__init__.py'),
ka_root.join('third_party/boto/auth.py'),
ka_root.join('third_party/vendored/__init__.py'),
ka_root.join('third_party/vendored/third_party/__init__.py'),
ka_root.join('third_party/vendored/third_party/idna/__init__.py'),
ka_root.join('third_party/vendored/third_party/idna/codec.py'),
ka_root.join('import_lint.py'),
'/usr/lib/python/lxml/__init__.py',
'/usr/lib/python/lxml/parser.py',
}
patcher = mock.patch('os.path.exists', lambda f: f in files)
self.addCleanup(patcher.stop)
patcher.start()
patcher = mock.patch('os.listdir', lambda d: ['idna', 'urllib3'])
self.addCleanup(patcher.stop)
patcher.start()
patcher = mock.patch(
'os.stat',
lambda f: mock.Mock(st_size=(0 if '__init__.py' in f else 1000)))
self.addCleanup(patcher.stop)
patcher.start()
patcher = mock.patch('sys.path', [ka_root.root,
ka_root.join('third_party/vendored'),
'/usr/lib/python'])
self.addCleanup(patcher.stop)
patcher.start()
def _lint(self, python_contents):
"""Superclass's assert_error() and assert_no_error call this."""
path = ka_root.join('foo.py')
self.set_file_contents(path, python_contents)
return jinja2_lint.lint_no_templatetags_calls_in_python([path])
def _lint(self, html_contents):
"""Superclass's assert_error() and assert_no_error call this."""
path = ka_root.join('templates', 'foo.html')
self.set_file_contents(path, html_contents)
return jinja2_lint.lint_javascript_in_html([path])
def test_missing_file(self):
actual = import_lint._get_import_lines(ka_root.join('notafile.py'))
self.assertEqual(None, actual)
