def Get(self):
if (not IsDevServer() and not fnmatch(
urlparse(self._request.host).netloc, '*.appspot.com')):
# Only allow patches on appspot URLs; it doesn't matter if appspot.com is
# XSS'ed, but it matters for chrome.com.
redirect_host = 'https://chrome-apps-doc.appspot.com'
logging.info('Redirecting from XSS-able host %s to %s' %
(self._request.host, redirect_host))
return Response.Redirect('%s/_patch/%s' %
(redirect_host, self._request.path))
path_with_issue = self._request.path.lstrip('/')
if '/' in path_with_issue:
issue, path_without_issue = path_with_issue.split('/', 1)
else:
return Response.NotFound(
'Malformed URL. It should look like ' +
'https://developer.chrome.com/_patch/12345/extensions/...')
try:
response = RenderServlet(
Request(path_without_issue, self._request.host,
self._request.headers),
_PatchServletDelegate(issue, self._delegate)).Get()
# Disable cache for patched content.
response.headers.pop('cache-control', None)
except RietveldPatcherError as e:
response = Response.NotFound(e.message,
{'Content-Type': 'text/plain'})
redirect_url, permanent = response.GetRedirect()
if redirect_url is not None:
response = Response.Redirect(
'/_patch/%s%s' % (issue, redirect_url), permanent)
return response
def DelMulti(self, keys):
futures = []
for key in keys:
futures.append(db.delete_async(
PersistentObjectStoreItem.CreateKey(self._namespace, key)))
# If running the dev server, the futures don't complete until the server is
# *quitting*. This is annoying. Flush now.
if IsDevServer():
[future.wait() for future in futures]
def SetMulti(self, mapping):
futures = []
for key, value in mapping.items():
futures.append(
db.put_async(
PersistentObjectStoreItem.CreateItem(
self._namespace, key, value)))
# If running the dev server, the futures don't complete until the server is
# *quitting*. This is annoying. Flush now.
if IsDevServer():
[future.wait() for future in futures]
def SetMulti(self, mapping):
entities = [PersistentObjectStoreItem.CreateItem(
self._namespace, key, value)
for key, value in mapping.iteritems()]
# Some entites may be None if they were too large to insert. Skip those.
rpcs = [db.put_async(entity for entity in entities if entity)]
# If running the dev server, the futures don't complete until the server is
# *quitting*. This is annoying. Flush now.
if IsDevServer():
[rpc.wait() for rpc in rpcs]
return All(Future(callback=lambda: rpc.get_result()) for rpc in rpcs)
def SetMulti(self, mapping):
rpcs = [
db.put_async(
PersistentObjectStoreItem.CreateItem(self._namespace, key,
value))
for key, value in mapping.iteritems()
]
# If running the dev server, the futures don't complete until the server is
# *quitting*. This is annoying. Flush now.
if IsDevServer():
[rpc.wait() for rpc in rpcs]
return All(Future(callback=lambda: rpc.get_result()) for rpc in rpcs)
def CreateServerInstance(self):
object_store_creator = ObjectStoreCreator(start_empty=False)
branch_utility = self._delegate.CreateBranchUtility(
object_store_creator)
# In production have offline=True so that we can catch cron errors. In
# development it's annoying to have to run the cron job, so offline=False.
host_file_system_provider = self._delegate.CreateHostFileSystemProvider(
object_store_creator, offline=not IsDevServer())
github_file_system_provider = self._delegate.CreateGithubFileSystemProvider(
object_store_creator)
return ServerInstance(object_store_creator,
CompiledFileSystem.Factory(object_store_creator),
branch_utility, host_file_system_provider,
github_file_system_provider)
def Get(self):
if not IsDevServer():
return Response.BadRequest('')
import cPickle
from persistent_object_store_appengine import PersistentObjectStoreAppengine
with open(self._request.path, 'r') as f:
data = cPickle.load(f)
for namespace, contents in data.iteritems():
store = PersistentObjectStoreAppengine(namespace)
for k, v in cPickle.loads(contents).iteritems():
try:
store.Set(k, v).Get()
except:
logging.warn('Skipping entry %s because of errors.' % k)
return Response.Ok('Data pushed!')
def GetSamplesModel(self, platform):
if self._platform_data[platform].samples_model is None:
# Note: samples are super slow in the dev server because it doesn't
# support async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
app_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = self._host_fs_at_master
# TODO(kalman): Re-enable the apps samples, see http://crbug.com/344097.
app_samples_fs = EmptyDirFileSystem()
self._platform_data[platform].samples_model = SamplesModel(
extension_samples_fs,
app_samples_fs, self._compiled_fs_factory,
self.GetReferenceResolver(platform), self._base_path, platform)
return self._platform_data[platform].samples_model
def CreateServerInstance(self):
object_store_creator = ObjectStoreCreator(start_empty=False)
branch_utility = self._delegate.CreateBranchUtility(
object_store_creator)
# In production have offline=True so that we can catch cron errors. In
# development it's annoying to have to run the cron job, so offline=False.
# Note that offline=True if running on any appengine server due to
# http://crbug.com/345361.
host_file_system_provider = self._delegate.CreateHostFileSystemProvider(
object_store_creator,
offline=not (IsDevServer() or IsReleaseServer()))
github_file_system_provider = self._delegate.CreateGithubFileSystemProvider(
object_store_creator)
return ServerInstance(
object_store_creator,
CompiledFileSystem.Factory(object_store_creator), branch_utility,
host_file_system_provider, github_file_system_provider,
CloudStorageFileSystemProvider(object_store_creator))
def Create(cls, branch='master', commit=None):
token, _ = app_identity.get_access_token(GITILES_OAUTH2_SCOPE)
# Log the access token (once per token) so that it can be sneakily re-used
# in development.
if token not in cls._logged_tokens:
logging.info('Got token %s for scope %s' % (token, GITILES_OAUTH2_SCOPE))
cls._logged_tokens.add(token)
# Only include forced-auth (/a/) in the Gitiles URL if we have a token and
# this is not the development server.
path_prefix = ('' if token is None or IsDevServer()
else _AUTH_PATH_PREFIX)
if commit:
base_url = '%s%s/%s/%s' % (
GITILES_BASE, path_prefix, GITILES_SRC_ROOT, commit)
elif branch is 'master':
base_url = '%s%s/%s/master' % (
GITILES_BASE, path_prefix, GITILES_SRC_ROOT)
else:
base_url = '%s%s/%s/%s/%s' % (
GITILES_BASE, path_prefix, GITILES_SRC_ROOT,
GITILES_BRANCHES_PATH, branch)
return GitilesFileSystem(AppEngineUrlFetcher(), base_url, branch, commit)
def __init__(self,
object_store_creator,
compiled_fs_factory,
branch_utility,
host_file_system_provider,
github_file_system_provider,
gcs_file_system_provider,
base_path='/'):
'''
|object_store_creator|
The ObjectStoreCreator used to create almost all caches.
|compiled_fs_factory|
Factory used to create CompiledFileSystems, a higher-level cache type
than ObjectStores. This can usually be derived from just
|object_store_creator| but under special circumstances a different
implementation needs to be passed in.
|branch_utility|
Has knowledge of Chrome branches, channels, and versions.
|host_file_system_provider|
Creates FileSystem instances which host the server at alternative
revisions.
|github_file_system_provider|
Creates FileSystem instances backed by GitHub.
|base_path|
The path which all HTML is generated relative to. Usually this is /
but some servlets need to override this.
'''
self.object_store_creator = object_store_creator
self.compiled_fs_factory = compiled_fs_factory
self.host_file_system_provider = host_file_system_provider
host_fs_at_trunk = host_file_system_provider.GetTrunk()
self.github_file_system_provider = github_file_system_provider
self.gcs_file_system_provider = gcs_file_system_provider
assert base_path.startswith('/') and base_path.endswith('/')
self.base_path = base_path
self.host_file_system_iterator = HostFileSystemIterator(
host_file_system_provider,
branch_utility)
self.features_bundle = FeaturesBundle(
host_fs_at_trunk,
self.compiled_fs_factory,
self.object_store_creator)
self.api_models = APIModels(
self.features_bundle,
self.compiled_fs_factory,
host_fs_at_trunk)
self.availability_finder = AvailabilityFinder(
branch_utility,
compiled_fs_factory,
self.host_file_system_iterator,
host_fs_at_trunk,
object_store_creator)
self.api_categorizer = APICategorizer(
host_fs_at_trunk,
compiled_fs_factory)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_fs_factory,
host_fs_at_trunk,
self.availability_finder,
self.api_models,
self.features_bundle,
self.object_store_creator)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory,
self.api_models,
object_store_creator)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
app_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = host_fs_at_trunk
# TODO(kalman): Re-enable the apps samples, see http://crbug.com/344097.
app_samples_fs = EmptyDirFileSystem()
#app_samples_fs = github_file_system_provider.Create(
# 'GoogleChrome', 'chrome-app-samples')
self.samples_data_source_factory = SamplesDataSource.Factory(
extension_samples_fs,
app_samples_fs,
CompiledFileSystem.Factory(object_store_creator),
self.ref_resolver_factory,
base_path)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.content_providers = ContentProviders(
object_store_creator,
self.compiled_fs_factory,
host_fs_at_trunk,
self.github_file_system_provider,
self.gcs_file_system_provider)
# TODO(kalman): Move all the remaining DataSources into DataSourceRegistry,
# then factor out the DataSource creation into a factory method, so that
# the entire ServerInstance doesn't need to be passed in here.
self.template_renderer = TemplateRenderer(self)
# TODO(kalman): It may be better for |document_renderer| to construct a
# TemplateDataSource itself rather than depending on template_renderer, but
# for that the above todo should be addressed.
self.document_renderer = DocumentRenderer(
TableOfContentsRenderer(host_fs_at_trunk,
compiled_fs_factory,
self.template_renderer),
self.ref_resolver_factory.Create())
def _GetImpl(self):
# Cron strategy:
#
# Find all public template files and static files, and render them. Most of
# the time these won't have changed since the last cron run, so it's a
# little wasteful, but hopefully rendering is really fast (if it isn't we
# have a problem).
_cronlog.info('starting')
# This is returned every time RenderServlet wants to create a new
# ServerInstance.
#
# TODO(kalman): IMPORTANT. This sometimes throws an exception, breaking
# everything. Need retry logic at the fetcher level.
server_instance = self._GetSafeServerInstance()
trunk_fs = server_instance.host_file_system_provider.GetTrunk()
def render(path):
request = Request(path, self._request.host, self._request.headers)
delegate = _SingletonRenderServletDelegate(server_instance)
return RenderServlet(request, delegate).Get()
def request_files_in_dir(path, prefix='', strip_ext=None):
'''Requests every file found under |path| in this host file system, with
a request prefix of |prefix|. |strip_ext| is an optional list of file
extensions that should be stripped from paths before requesting.
'''
def maybe_strip_ext(name):
if name == SITE_VERIFICATION_FILE or not strip_ext:
return name
base, ext = posixpath.splitext(name)
return base if ext in strip_ext else name
files = [
maybe_strip_ext(name)
for name, _ in CreateURLsFromPaths(trunk_fs, path, prefix)
]
return _RequestEachItem(path, files, render)
results = []
try:
# Start running the hand-written Cron methods first; they can be run in
# parallel. They are resolved at the end.
def run_cron_for_future(target):
title = target.__class__.__name__
future, init_timer = TimerClosure(target.Cron)
assert isinstance(
future,
Future), ('%s.Cron() did not return a Future' % title)
def resolve():
resolve_timer = Timer()
try:
future.Get()
except Exception as e:
_cronlog.error('%s: error %s' %
(title, traceback.format_exc()))
results.append(False)
if IsDeadlineExceededError(e): raise
finally:
resolve_timer.Stop()
_cronlog.info(
'%s took %s: %s to initialize and %s to resolve' %
(title,
init_timer.With(resolve_timer).FormatElapsed(),
init_timer.FormatElapsed(),
resolve_timer.FormatElapsed()))
return Future(delegate=Gettable(resolve))
targets = (CreateDataSources(server_instance).values() +
[server_instance.content_providers])
title = 'initializing %s parallel Cron targets' % len(targets)
_cronlog.info(title)
timer = Timer()
try:
cron_futures = [
run_cron_for_future(target) for target in targets
]
finally:
_cronlog.info('%s took %s' %
(title, timer.Stop().FormatElapsed()))
# Rendering the public templates will also pull in all of the private
# templates.
results.append(
request_files_in_dir(PUBLIC_TEMPLATES,
strip_ext=('.html', '.md')))
# Rendering the public templates will have pulled in the .js and
# manifest.json files (for listing examples on the API reference pages),
# but there are still images, CSS, etc.
results.append(request_files_in_dir(STATIC_DOCS, prefix='static'))
# Samples are too expensive to run on the dev server, where there is no
# parallel fetch.
if not IsDevServer():
# Fetch each individual sample file.
results.append(
request_files_in_dir(EXAMPLES,
prefix='extensions/examples'))
# Fetch the zip file of each example (contains all the individual
# files).
example_zips = []
for root, _, files in trunk_fs.Walk(EXAMPLES):
example_zips.extend(root + '.zip' for name in files
if name == 'manifest.json')
results.append(
_RequestEachItem(
'example zips', example_zips,
lambda path: render('extensions/examples/' + path)))
# Resolve the hand-written Cron method futures.
title = 'resolving %s parallel Cron targets' % len(targets)
_cronlog.info(title)
timer = Timer()
try:
for future in cron_futures:
future.Get()
finally:
_cronlog.info('%s took %s' %
(title, timer.Stop().FormatElapsed()))
except:
results.append(False)
# This should never actually happen (each cron step does its own
# conservative error checking), so re-raise no matter what it is.
_cronlog.error('uncaught error: %s' % traceback.format_exc())
raise
finally:
success = all(results)
_cronlog.info('finished (%s)', 'success' if success else 'FAILED')
return (Response.Ok('Success')
if success else Response.InternalError('Failure'))
def __init__(self,
object_store_creator,
compiled_fs_factory,
branch_utility,
host_file_system_provider,
github_file_system_provider,
base_path='/'):
'''
|object_store_creator|
The ObjectStoreCreator used to create almost all caches.
|compiled_fs_factory|
Factory used to create CompiledFileSystems, a higher-level cache type
than ObjectStores. This can usually be derived from just
|object_store_creator| but under special circumstances a different
implementation needs to be passed in.
|branch_utility|
Has knowledge of Chrome branches, channels, and versions.
|host_file_system_provider|
Creates FileSystem instances which host the server at alternative
revisions.
|github_file_system_provider|
Creates FileSystem instances backed by GitHub.
|base_path|
The path which all HTML is generated relative to. Usually this is /
but some servlets need to override this.
'''
self.object_store_creator = object_store_creator
self.compiled_fs_factory = compiled_fs_factory
self.host_file_system_provider = host_file_system_provider
host_fs_at_trunk = host_file_system_provider.GetTrunk()
self.github_file_system_provider = github_file_system_provider
assert base_path.startswith('/') and base_path.endswith('/')
self.base_path = base_path
self.host_file_system_iterator = HostFileSystemIterator(
host_file_system_provider, branch_utility)
self.features_bundle = FeaturesBundle(host_fs_at_trunk,
self.compiled_fs_factory,
self.object_store_creator)
self.api_models = APIModels(self.features_bundle,
self.compiled_fs_factory, host_fs_at_trunk)
self.availability_finder = AvailabilityFinder(
branch_utility, compiled_fs_factory,
self.host_file_system_iterator, host_fs_at_trunk,
object_store_creator)
self.api_list_data_source_factory = APIListDataSource.Factory(
self.compiled_fs_factory, host_fs_at_trunk, self.features_bundle,
self.object_store_creator)
self.api_data_source_factory = APIDataSource.Factory(
self.compiled_fs_factory, host_fs_at_trunk, svn_constants.API_PATH,
self.availability_finder, branch_utility)
self.ref_resolver_factory = ReferenceResolver.Factory(
self.api_data_source_factory, self.api_models,
object_store_creator)
self.api_data_source_factory.SetReferenceResolverFactory(
self.ref_resolver_factory)
# Note: samples are super slow in the dev server because it doesn't support
# async fetch, so disable them.
if IsDevServer():
extension_samples_fs = EmptyDirFileSystem()
app_samples_fs = EmptyDirFileSystem()
else:
extension_samples_fs = host_fs_at_trunk
app_samples_fs = github_file_system_provider.Create(
'GoogleChrome', 'chrome-app-samples')
self.samples_data_source_factory = SamplesDataSource.Factory(
extension_samples_fs, app_samples_fs,
CompiledFileSystem.Factory(object_store_creator),
self.ref_resolver_factory, svn_constants.EXAMPLES_PATH, base_path)
self.api_data_source_factory.SetSamplesDataSourceFactory(
self.samples_data_source_factory)
self.intro_data_source_factory = IntroDataSource.Factory(
self.compiled_fs_factory, host_fs_at_trunk,
self.ref_resolver_factory,
[svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH])
self.path_canonicalizer = PathCanonicalizer(self.compiled_fs_factory,
host_fs_at_trunk)
self.content_providers = ContentProviders(
self.compiled_fs_factory, host_fs_at_trunk,
self.github_file_system_provider)
# TODO(kalman): Move all the remaining DataSources into DataSourceRegistry,
# then factor out the DataSource creation into a factory method, so that
# the entire ServerInstance doesn't need to be passed in here.
self.template_renderer = TemplateRenderer(self)
self.strings_json_path = svn_constants.STRINGS_JSON_PATH
self.manifest_json_path = svn_constants.MANIFEST_JSON_PATH
self.manifest_features_path = svn_constants.MANIFEST_FEATURES_PATH