def Get(self):
if (not IsDevServer() and not fnmatch(
urlparse(self._request.host).netloc, '*.appspot.com')):
# Only allow patches on appspot URLs; it doesn't matter if appspot.com is
# XSS'ed, but it matters for chrome.com.
redirect_host = 'https://chrome-apps-doc.appspot.com'
logging.info('Redirecting from XSS-able host %s to %s' %
(self._request.host, redirect_host))
return Response.Redirect('%s/_patch/%s' %
(redirect_host, self._request.path))
path_with_issue = self._request.path.lstrip('/')
if '/' in path_with_issue:
issue, path_without_issue = path_with_issue.split('/', 1)
else:
return Response.NotFound(
'Malformed URL. It should look like ' +
'https://developer.chrome.com/_patch/12345/extensions/...')
try:
response = RenderServlet(
Request(path_without_issue, self._request.host,
self._request.headers),
_PatchServletDelegate(issue, self._delegate)).Get()
# Disable cache for patched content.
response.headers.pop('cache-control', None)
except RietveldPatcherError as e:
response = Response.NotFound(e.message,
{'Content-Type': 'text/plain'})
redirect_url, permanent = response.GetRedirect()
if redirect_url is not None:
response = Response.Redirect(
'/_patch/%s%s' % (issue, redirect_url), permanent)
return response
def DelMulti(self, keys):
futures = []
for key in keys:
futures.append(
db.delete_async(
PersistentObjectStoreItem.CreateKey(self._namespace, key)))
# If running the dev server, the futures don't complete until the server is
# *quitting*. This is annoying. Flush now.
if IsDevServer():
[future.wait() for future in futures]
def SetMulti(self, mapping):
futures = []
for key, value in mapping.items():
futures.append(
db.put_async(
PersistentObjectStoreItem.CreateItem(
self._namespace, key, value)))
# If running the dev server, the futures don't complete until the server is
# *quitting*. This is annoying. Flush now.
if IsDevServer():
[future.wait() for future in futures]
def _IsSamplesDisabled():
return IsDevServer()
def _GetImpl(self):
# Cron strategy:
#
# Find all public template files and static files, and render them. Most of
# the time these won't have changed since the last cron run, so it's a
# little wasteful, but hopefully rendering is really fast (if it isn't we
# have a problem).
logging.info('cron: starting')
# This is returned every time RenderServlet wants to create a new
# ServerInstance.
server_instance = self._GetSafeServerInstance()
def get_via_render_servlet(path):
request = Request(path, self._request.host, self._request.headers)
delegate = _SingletonRenderServletDelegate(server_instance)
return RenderServlet(request, delegate).Get()
def run_cron_for_dir(d, path_prefix=''):
success = True
start_time = time.time()
files = dict(
CreateURLsFromPaths(server_instance.host_file_system, d,
path_prefix))
logging.info('cron: rendering %s files from %s...' %
(len(files), d))
try:
for i, path in enumerate(files):
error = None
try:
response = get_via_render_servlet(path)
if response.status != 200:
error = 'Got %s response' % response.status
except DeadlineExceededError:
logging.error(
'cron: deadline exceeded rendering %s (%s of %s): %s'
%
(path, i + 1, len(files), traceback.format_exc()))
raise
except error:
pass
if error:
logging.error('cron: error rendering %s: %s' %
(path, error))
success = False
finally:
logging.info(
'cron: rendering %s files from %s took %s seconds' %
(len(files), d, time.time() - start_time))
return success
success = True
try:
# Render all of the publicly accessible files.
cron_runs = [
# Note: rendering the public templates will pull in all of the private
# templates.
(svn_constants.PUBLIC_TEMPLATE_PATH, ''),
# Note: rendering the public templates will have pulled in the .js
# and manifest.json files (for listing examples on the API reference
# pages), but there are still images, CSS, etc.
(svn_constants.STATIC_PATH, 'static/'),
]
if not IsDevServer():
cron_runs.append(
(svn_constants.EXAMPLES_PATH, 'extensions/examples/'))
# Note: don't try to short circuit any of this stuff. We want to run
# the cron for all the directories regardless of intermediate
# failures.
for path, path_prefix in cron_runs:
success = run_cron_for_dir(path,
path_prefix=path_prefix) and success
# TODO(kalman): Generic way for classes to request cron access. The next
# two special cases are ugly. It would potentially greatly speed up cron
# runs, too.
# Extension examples have zip files too. Well, so do apps, but the app
# file system doesn't get the Offline treatment so they don't need cron.
if not IsDevServer():
manifest_json = 'manifest.json'
example_zips = []
for root, _, files in server_instance.host_file_system.Walk(
svn_constants.EXAMPLES_PATH):
example_zips.extend(root + '.zip' for name in files
if name == manifest_json)
logging.info('cron: rendering %s example zips...' %
len(example_zips))
start_time = time.time()
try:
success = success and all(
get_via_render_servlet('extensions/examples/%s' %
z).status == 200
for z in example_zips)
finally:
logging.info(
'cron: rendering %s example zips took %s seconds' %
(len(example_zips), time.time() - start_time))
except DeadlineExceededError:
success = False
logging.info('cron: running Redirector cron...')
server_instance.redirector.Cron()
logging.info('cron: finished (%s)' %
('success' if success else 'failure', ))
return (Response.Ok('Success')
if success else Response.InternalError('Failure'))
def CreateAppSamplesFileSystem(self, object_store_creator):
# TODO(kalman): CachingFileSystem wrapper for GithubFileSystem, but it's
# not supported yet (see comment there).
return (EmptyDirFileSystem() if IsDevServer() else
GithubFileSystem.Create(object_store_creator))
def __init__(self, channel, object_store_creator, host_file_system,
app_samples_file_system, base_path, compiled_fs_factory):
self.channel = channel
self.object_store_creator = object_store_creator
self.host_file_system = host_file_system
self.app_samples_file_system = app_samples_file_system
self.compiled_host_fs_factory = compiled_fs_factory
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.API_PATH,
svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.API_PATH)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = self.host_file_system
self.samples_data_source_factory = SamplesDataSource.Factory(
channel, extension_samples_fs,
CompiledFileSystem.Factory(extension_samples_fs,
object_store_creator),
self.app_samples_file_system,
CompiledFileSystem.Factory(self.app_samples_file_system,
object_store_creator),
self.ref_resolver_factory, svn_constants.EXAMPLES_PATH)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_host_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.JSON_PATH, base_path)
self.template_data_source_factory = TemplateDataSource.Factory(
channel, self.api_data_source_factory,
self.api_list_data_source_factory, self.intro_data_source_factory,
self.samples_data_source_factory, self.sidenav_data_source_factory,
self.compiled_host_fs_factory, self.ref_resolver_factory,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH, base_path)
self.example_zipper = ExampleZipper(self.compiled_host_fs_factory,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(
channel, self.compiled_host_fs_factory)
self.content_cache = self.compiled_host_fs_factory.CreateIdentity(
ServerInstance)
def __init__(self, object_store_creator, host_file_system,
app_samples_file_system, base_path, compiled_fs_factory,
branch_utility, host_file_system_creator):
self.object_store_creator = object_store_creator
self.host_file_system = host_file_system
self.app_samples_file_system = app_samples_file_system
self.compiled_host_fs_factory = compiled_fs_factory
self.host_file_system_creator = host_file_system_creator
self.availability_finder_factory = AvailabilityFinder.Factory(
object_store_creator, self.compiled_host_fs_factory,
branch_utility, host_file_system_creator)
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_host_fs_factory, self.host_file_system,
svn_constants.API_PATH, svn_constants.PUBLIC_TEMPLATE_PATH)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.API_PATH,
self.availability_finder_factory)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
object_store_creator)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = self.host_file_system
self.samples_data_source_factory = SamplesDataSource.Factory(
extension_samples_fs,
CompiledFileSystem.Factory(extension_samples_fs,
object_store_creator),
self.app_samples_file_system,
CompiledFileSystem.Factory(self.app_samples_file_system,
object_store_creator),
self.ref_resolver_factory, svn_constants.EXAMPLES_PATH, base_path)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_host_fs_factory, self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.sidenav_data_source_factory = SidenavDataSource.Factory(
self.compiled_host_fs_factory, svn_constants.JSON_PATH)
self.manifest_data_source = ManifestDataSource(
self.compiled_host_fs_factory, host_file_system, '/'.join(
(svn_constants.JSON_PATH, 'manifest.json')), '/'.join(
(svn_constants.API_PATH, '_manifest_features.json')))
self.template_data_source_factory = TemplateDataSource.Factory(
self.api_data_source_factory, self.api_list_data_source_factory,
self.intro_data_source_factory, self.samples_data_source_factory,
self.sidenav_data_source_factory, self.compiled_host_fs_factory,
self.ref_resolver_factory, self.manifest_data_source,
svn_constants.PUBLIC_TEMPLATE_PATH,
svn_constants.PRIVATE_TEMPLATE_PATH, base_path)
self.api_data_source_factory.SetTemplateDataSource(
self.template_data_source_factory)
self.example_zipper = ExampleZipper(self.compiled_host_fs_factory,
self.host_file_system,
svn_constants.DOCS_PATH)
self.path_canonicalizer = PathCanonicalizer(
self.compiled_host_fs_factory)
self.redirector = Redirector(self.compiled_host_fs_factory,
self.host_file_system,
svn_constants.PUBLIC_TEMPLATE_PATH)
def _GetImpl(self):
# Cron strategy:
#
# Find all public template files and static files, and render them. Most of
# the time these won't have changed since the last cron run, so it's a
# little wasteful, but hopefully rendering is really fast (if it isn't we
# have a problem).
channel = self._channel
logging.info('cron/%s: starting' % channel)
# This is returned every time RenderServlet wants to create a new
# ServerInstance.
server_instance = self._GetSafeServerInstance()
def get_via_render_servlet(path):
return RenderServlet(
Request(path, self._request.host, self._request.headers),
_SingletonRenderServletDelegate(server_instance)).Get()
def run_cron_for_dir(d, path_prefix=''):
success = True
start_time = time.time()
files = [f for f in server_instance.content_cache.GetFromFileListing(d)
if not f.endswith('/')]
logging.info('cron/%s: rendering %s files from %s...' % (
channel, len(files), d))
try:
for i, f in enumerate(files):
error = None
path = '%s%s' % (path_prefix, f)
try:
response = get_via_render_servlet(path)
if response.status != 200:
error = 'Got %s response' % response.status
except DeadlineExceededError:
logging.error(
'cron/%s: deadline exceeded rendering %s (%s of %s): %s' % (
channel, path, i + 1, len(files), traceback.format_exc()))
raise
except error:
pass
if error:
logging.error('cron/%s: error rendering %s: %s' % (
channel, path, error))
success = False
finally:
logging.info('cron/%s: rendering %s files from %s took %s seconds' % (
channel, len(files), d, time.time() - start_time))
return success
success = True
try:
# Render all of the publicly accessible files.
cron_runs = [
# Note: rendering the public templates will pull in all of the private
# templates.
(svn_constants.PUBLIC_TEMPLATE_PATH, ''),
# Note: rendering the public templates will have pulled in the .js
# and manifest.json files (for listing examples on the API reference
# pages), but there are still images, CSS, etc.
(svn_constants.STATIC_PATH, 'static/'),
]
if not IsDevServer():
cron_runs.append(
(svn_constants.EXAMPLES_PATH, 'extensions/examples/'))
# Note: don't try to short circuit any of this stuff. We want to run
# the cron for all the directories regardless of intermediate
# failures.
for path, path_prefix in cron_runs:
success = run_cron_for_dir(path, path_prefix=path_prefix) and success
# TODO(kalman): Generic way for classes to request cron access. The next
# two special cases are ugly. It would potentially greatly speed up cron
# runs, too.
# Extension examples have zip files too. Well, so do apps, but the app
# file system doesn't get the Offline treatment so they don't need cron.
if not IsDevServer():
manifest_json = '/manifest.json'
example_zips = [
'%s.zip' % filename[:-len(manifest_json)]
for filename in server_instance.content_cache.GetFromFileListing(
svn_constants.EXAMPLES_PATH)
if filename.endswith(manifest_json)]
logging.info('cron/%s: rendering %s example zips...' % (
channel, len(example_zips)))
start_time = time.time()
try:
success = success and all(
get_via_render_servlet('extensions/examples/%s' % z).status == 200
for z in example_zips)
finally:
logging.info('cron/%s: rendering %s example zips took %s seconds' % (
channel, len(example_zips), time.time() - start_time))
# Also trigger a redirect so that PathCanonicalizer has an opportunity to
# cache file listings.
logging.info('cron/%s: triggering a redirect...' % channel)
redirect_response = get_via_render_servlet('storage.html')
success = success and redirect_response.status == 302
except DeadlineExceededError:
success = False
logging.info('cron/%s: finished' % channel)
return (Response.Ok('Success') if success else
Response.InternalError('Failure'))
from fnmatch import fnmatch
import logging
import mimetypes
import os
import traceback
from appengine_wrappers import IsDevServer
from branch_utility import BranchUtility
from file_system import FileNotFoundError
from server_instance import ServerInstance
from servlet import Servlet, Response
import svn_constants
_DEFAULT_CHANNEL = 'stable'
_ALWAYS_ONLINE = IsDevServer()
def _IsBinaryMimetype(mimetype):
return any(
mimetype.startswith(prefix) for prefix in ['audio', 'image', 'video'])
def AlwaysOnline(fn):
'''A function decorator which forces the rendering to be always online rather
than the default offline behaviour. Useful for testing.
'''
def impl(*args, **optargs):
global _ALWAYS_ONLINE
was_always_online = _ALWAYS_ONLINE
try:
class Handler(webapp.RequestHandler):
# AppEngine instances should never need to call out to SVN. That should only
# ever be done by the cronjobs, which then write the result into DataStore,
# which is as far as instances look.
#
# Why? SVN is slow and a bit flaky. Cronjobs failing is annoying but
# temporary. Instances failing affects users, and is really bad.
#
# Anyway - to enforce this, we actually don't give instances access to SVN.
# If anything is missing from datastore, it'll be a 404. If the cronjobs
# don't manage to catch everything - uhoh. On the other hand, we'll figure it
# out pretty soon, and it also means that legitimate 404s are caught before a
# round trip to SVN.
#
# However, we can't expect users of preview.py nor the dev server to run a
# cronjob first, so, this is a hack allow that to be online all of the time.
# TODO(kalman): achieve this via proper dependency injection.
ALWAYS_ONLINE = IsDevServer()
def __init__(self, request, response):
super(Handler, self).__init__(request, response)
def _HandleGet(self, path):
channel_name, real_path = BranchUtility.SplitChannelNameFromPath(path)
if channel_name == _DEFAULT_CHANNEL:
self.redirect('/%s' % real_path)
return
if channel_name is None:
channel_name = _DEFAULT_CHANNEL
# TODO(kalman): Check if |path| is a directory and serve path/index.html
# rather than special-casing apps/extensions.
if real_path.strip('/') == 'apps':
real_path = 'apps/index.html'
if real_path.strip('/') == 'extensions':
real_path = 'extensions/index.html'
constructor = (ServerInstance.CreateOnline if Handler.ALWAYS_ONLINE
else ServerInstance.GetOrCreateOffline)
server_instance = constructor(channel_name)
canonical_path = server_instance.path_canonicalizer.Canonicalize(
real_path)
if real_path != canonical_path:
self.redirect(canonical_path)
return
server_instance.Get(real_path, self.request, self.response)
def _HandleCron(self, path):
# Cron strategy:
#
# Find all public template files and static files, and render them. Most of
# the time these won't have changed since the last cron run, so it's a
# little wasteful, but hopefully rendering is really fast (if it isn't we
# have a problem).
class MockResponse(object):
def __init__(self):
self.status = 200
self.out = StringIO()
self.headers = {}
def set_status(self, status):
self.status = status
def clear(self, *args):
pass
class MockRequest(object):
def __init__(self, path):
self.headers = {}
self.path = path
self.url = '//localhost/%s' % path
channel = path.split('/')[-1]
logging.info('cron/%s: starting' % channel)
server_instance = ServerInstance.CreateOnline(channel)
def run_cron_for_dir(d, path_prefix=''):
success = True
start_time = time.time()
files = [
f for f in server_instance.content_cache.GetFromFileListing(d)
if not f.endswith('/')
]
logging.info('cron/%s: rendering %s files from %s...' %
(channel, len(files), d))
for i, f in enumerate(files):
error = None
path = '%s%s' % (path_prefix, f)
try:
response = MockResponse()
server_instance.Get(path, MockRequest(path), response)
if response.status != 200:
error = 'Got %s response' % response.status
except DeadlineExceededError:
logging.error(
'cron/%s: deadline exceeded rendering %s (%s of %s): %s'
% (channel, path, i + 1, len(files),
traceback.format_exc()))
raise
except error:
pass
if error:
logging.error('cron/%s: error rendering %s: %s' %
(channel, path, error))
success = False
logging.info(
'cron/%s: rendering %s files from %s took %s seconds' %
(channel, len(files), d, time.time() - start_time))
return success
success = True
for path, path_prefix in (
# Note: rendering the public templates will pull in all of the private
# templates.
(svn_constants.PUBLIC_TEMPLATE_PATH, ''),
# Note: rendering the public templates will have pulled in the .js and
# manifest.json files (for listing examples on the API reference pages),
# but there are still images, CSS, etc.
(svn_constants.STATIC_PATH, 'static/'),
(svn_constants.EXAMPLES_PATH, 'extensions/examples/')):
try:
# Note: don't try to short circuit any of this stuff. We want to run
# the cron for all the directories regardless of intermediate failures.
success = run_cron_for_dir(path,
path_prefix=path_prefix) and success
except DeadlineExceededError:
success = False
break
if success:
self.response.status = 200
self.response.out.write('Success')
else:
self.response.status = 500
self.response.out.write('Failure')
logging.info('cron/%s: finished' % channel)
def _RedirectSpecialCases(self, path):
google_dev_url = 'http://developer.google.com/chrome'
if path == '/' or path == '/index.html':
self.redirect(google_dev_url)
return True
if path == '/apps.html':
self.redirect('/apps/about_apps.html')
return True
return False
def _RedirectFromCodeDotGoogleDotCom(self, path):
if (not self.request.url.startswith(
('http://code.google.com', 'https://code.google.com'))):
return False
new_url = 'http://developer.chrome.com/'
# switch to https if necessary
if (self.request.url.startswith('https')):
new_url = new_url.replace('http', 'https', 1)
path = path.split('/')
if len(path) > 0 and path[0] == 'chrome':
path.pop(0)
for channel in BranchUtility.GetAllBranchNames():
if channel in path:
position = path.index(channel)
path.pop(position)
path.insert(0, channel)
new_url += '/'.join(path)
self.redirect(new_url)
return True
def get(self):
path = self.request.path
if path in ['favicon.ico', 'robots.txt']:
response.set_status(404)
return
if self._RedirectSpecialCases(path):
return
if path.startswith('/cron'):
# Crons often time out, and when they do *and* then eventually try to
# flush logs they die. Turn off autoflush and manually do so at the end.
logservice.AUTOFLUSH_ENABLED = False
try:
self._HandleCron(path)
finally:
logservice.flush()
return
# Redirect paths like "directory" to "directory/". This is so relative
# file paths will know to treat this as a directory.
if os.path.splitext(path)[1] == '' and path[-1] != '/':
self.redirect(path + '/')
return
path = path.strip('/')
if self._RedirectFromCodeDotGoogleDotCom(path):
return
self._HandleGet(path)
def CreateAppSamplesFileSystem(self, object_store_creator):
# TODO(kalman): OfflineServerInstance wrapper for GithubFileSystem, but
# the cron job doesn't crawl the samples yet.
return (EmptyDirFileSystem() if IsDevServer() else
GithubFileSystem.Create(object_store_creator))
