def Get(self): if (not IsDevServer() and not fnmatch( urlparse(self._request.host).netloc, '*.appspot.com')): # Only allow patches on appspot URLs; it doesn't matter if appspot.com is # XSS'ed, but it matters for chrome.com. redirect_host = 'https://chrome-apps-doc.appspot.com' logging.info('Redirecting from XSS-able host %s to %s' % (self._request.host, redirect_host)) return Response.Redirect('%s/_patch/%s' % (redirect_host, self._request.path)) path_with_issue = self._request.path.lstrip('/') if '/' in path_with_issue: issue, path_without_issue = path_with_issue.split('/', 1) else: return Response.NotFound( 'Malformed URL. It should look like ' + 'https://developer.chrome.com/_patch/12345/extensions/...') try: response = RenderServlet( Request(path_without_issue, self._request.host, self._request.headers), _PatchServletDelegate(issue, self._delegate)).Get() # Disable cache for patched content. response.headers.pop('cache-control', None) except RietveldPatcherError as e: response = Response.NotFound(e.message, {'Content-Type': 'text/plain'}) redirect_url, permanent = response.GetRedirect() if redirect_url is not None: response = Response.Redirect( '/_patch/%s%s' % (issue, redirect_url), permanent) return response
def DelMulti(self, keys): futures = [] for key in keys: futures.append( db.delete_async( PersistentObjectStoreItem.CreateKey(self._namespace, key))) # If running the dev server, the futures don't complete until the server is # *quitting*. This is annoying. Flush now. if IsDevServer(): [future.wait() for future in futures]
def SetMulti(self, mapping): futures = [] for key, value in mapping.items(): futures.append( db.put_async( PersistentObjectStoreItem.CreateItem( self._namespace, key, value))) # If running the dev server, the futures don't complete until the server is # *quitting*. This is annoying. Flush now. if IsDevServer(): [future.wait() for future in futures]
def _IsSamplesDisabled(): return IsDevServer()
def _GetImpl(self): # Cron strategy: # # Find all public template files and static files, and render them. Most of # the time these won't have changed since the last cron run, so it's a # little wasteful, but hopefully rendering is really fast (if it isn't we # have a problem). logging.info('cron: starting') # This is returned every time RenderServlet wants to create a new # ServerInstance. server_instance = self._GetSafeServerInstance() def get_via_render_servlet(path): request = Request(path, self._request.host, self._request.headers) delegate = _SingletonRenderServletDelegate(server_instance) return RenderServlet(request, delegate).Get() def run_cron_for_dir(d, path_prefix=''): success = True start_time = time.time() files = dict( CreateURLsFromPaths(server_instance.host_file_system, d, path_prefix)) logging.info('cron: rendering %s files from %s...' % (len(files), d)) try: for i, path in enumerate(files): error = None try: response = get_via_render_servlet(path) if response.status != 200: error = 'Got %s response' % response.status except DeadlineExceededError: logging.error( 'cron: deadline exceeded rendering %s (%s of %s): %s' % (path, i + 1, len(files), traceback.format_exc())) raise except error: pass if error: logging.error('cron: error rendering %s: %s' % (path, error)) success = False finally: logging.info( 'cron: rendering %s files from %s took %s seconds' % (len(files), d, time.time() - start_time)) return success success = True try: # Render all of the publicly accessible files. cron_runs = [ # Note: rendering the public templates will pull in all of the private # templates. (svn_constants.PUBLIC_TEMPLATE_PATH, ''), # Note: rendering the public templates will have pulled in the .js # and manifest.json files (for listing examples on the API reference # pages), but there are still images, CSS, etc. (svn_constants.STATIC_PATH, 'static/'), ] if not IsDevServer(): cron_runs.append( (svn_constants.EXAMPLES_PATH, 'extensions/examples/')) # Note: don't try to short circuit any of this stuff. We want to run # the cron for all the directories regardless of intermediate # failures. for path, path_prefix in cron_runs: success = run_cron_for_dir(path, path_prefix=path_prefix) and success # TODO(kalman): Generic way for classes to request cron access. The next # two special cases are ugly. It would potentially greatly speed up cron # runs, too. # Extension examples have zip files too. Well, so do apps, but the app # file system doesn't get the Offline treatment so they don't need cron. if not IsDevServer(): manifest_json = 'manifest.json' example_zips = [] for root, _, files in server_instance.host_file_system.Walk( svn_constants.EXAMPLES_PATH): example_zips.extend(root + '.zip' for name in files if name == manifest_json) logging.info('cron: rendering %s example zips...' % len(example_zips)) start_time = time.time() try: success = success and all( get_via_render_servlet('extensions/examples/%s' % z).status == 200 for z in example_zips) finally: logging.info( 'cron: rendering %s example zips took %s seconds' % (len(example_zips), time.time() - start_time)) except DeadlineExceededError: success = False logging.info('cron: running Redirector cron...') server_instance.redirector.Cron() logging.info('cron: finished (%s)' % ('success' if success else 'failure', )) return (Response.Ok('Success') if success else Response.InternalError('Failure'))
def CreateAppSamplesFileSystem(self, object_store_creator): # TODO(kalman): CachingFileSystem wrapper for GithubFileSystem, but it's # not supported yet (see comment there). return (EmptyDirFileSystem() if IsDevServer() else GithubFileSystem.Create(object_store_creator))
def __init__(self, channel, object_store_creator, host_file_system, app_samples_file_system, base_path, compiled_fs_factory): self.channel = channel self.object_store_creator = object_store_creator self.host_file_system = host_file_system self.app_samples_file_system = app_samples_file_system self.compiled_host_fs_factory = compiled_fs_factory self.api_list_data_source_factory = APIListDataSource.Factory( self.compiled_host_fs_factory, svn_constants.API_PATH, svn_constants.PUBLIC_TEMPLATE_PATH) self.api_data_source_factory = APIDataSource.Factory( self.compiled_host_fs_factory, svn_constants.API_PATH) self.ref_resolver_factory = ReferenceResolver.Factory( self.api_data_source_factory, self.api_list_data_source_factory, object_store_creator) self.api_data_source_factory.SetReferenceResolverFactory( self.ref_resolver_factory) # Note: samples are super slow in the dev server because it doesn't support # async fetch, so disable them. if IsDevServer(): extension_samples_fs = EmptyDirFileSystem() else: extension_samples_fs = self.host_file_system self.samples_data_source_factory = SamplesDataSource.Factory( channel, extension_samples_fs, CompiledFileSystem.Factory(extension_samples_fs, object_store_creator), self.app_samples_file_system, CompiledFileSystem.Factory(self.app_samples_file_system, object_store_creator), self.ref_resolver_factory, svn_constants.EXAMPLES_PATH) self.api_data_source_factory.SetSamplesDataSourceFactory( self.samples_data_source_factory) self.intro_data_source_factory = IntroDataSource.Factory( self.compiled_host_fs_factory, self.ref_resolver_factory, [svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH]) self.sidenav_data_source_factory = SidenavDataSource.Factory( self.compiled_host_fs_factory, svn_constants.JSON_PATH, base_path) self.template_data_source_factory = TemplateDataSource.Factory( channel, self.api_data_source_factory, self.api_list_data_source_factory, self.intro_data_source_factory, self.samples_data_source_factory, self.sidenav_data_source_factory, self.compiled_host_fs_factory, self.ref_resolver_factory, svn_constants.PUBLIC_TEMPLATE_PATH, svn_constants.PRIVATE_TEMPLATE_PATH, base_path) self.example_zipper = ExampleZipper(self.compiled_host_fs_factory, svn_constants.DOCS_PATH) self.path_canonicalizer = PathCanonicalizer( channel, self.compiled_host_fs_factory) self.content_cache = self.compiled_host_fs_factory.CreateIdentity( ServerInstance)
def __init__(self, object_store_creator, host_file_system, app_samples_file_system, base_path, compiled_fs_factory, branch_utility, host_file_system_creator): self.object_store_creator = object_store_creator self.host_file_system = host_file_system self.app_samples_file_system = app_samples_file_system self.compiled_host_fs_factory = compiled_fs_factory self.host_file_system_creator = host_file_system_creator self.availability_finder_factory = AvailabilityFinder.Factory( object_store_creator, self.compiled_host_fs_factory, branch_utility, host_file_system_creator) self.api_list_data_source_factory = APIListDataSource.Factory( self.compiled_host_fs_factory, self.host_file_system, svn_constants.API_PATH, svn_constants.PUBLIC_TEMPLATE_PATH) self.api_data_source_factory = APIDataSource.Factory( self.compiled_host_fs_factory, svn_constants.API_PATH, self.availability_finder_factory) self.ref_resolver_factory = ReferenceResolver.Factory( self.api_data_source_factory, self.api_list_data_source_factory, object_store_creator) self.api_data_source_factory.SetReferenceResolverFactory( self.ref_resolver_factory) # Note: samples are super slow in the dev server because it doesn't support # async fetch, so disable them. if IsDevServer(): extension_samples_fs = EmptyDirFileSystem() else: extension_samples_fs = self.host_file_system self.samples_data_source_factory = SamplesDataSource.Factory( extension_samples_fs, CompiledFileSystem.Factory(extension_samples_fs, object_store_creator), self.app_samples_file_system, CompiledFileSystem.Factory(self.app_samples_file_system, object_store_creator), self.ref_resolver_factory, svn_constants.EXAMPLES_PATH, base_path) self.api_data_source_factory.SetSamplesDataSourceFactory( self.samples_data_source_factory) self.intro_data_source_factory = IntroDataSource.Factory( self.compiled_host_fs_factory, self.ref_resolver_factory, [svn_constants.INTRO_PATH, svn_constants.ARTICLE_PATH]) self.sidenav_data_source_factory = SidenavDataSource.Factory( self.compiled_host_fs_factory, svn_constants.JSON_PATH) self.manifest_data_source = ManifestDataSource( self.compiled_host_fs_factory, host_file_system, '/'.join( (svn_constants.JSON_PATH, 'manifest.json')), '/'.join( (svn_constants.API_PATH, '_manifest_features.json'))) self.template_data_source_factory = TemplateDataSource.Factory( self.api_data_source_factory, self.api_list_data_source_factory, self.intro_data_source_factory, self.samples_data_source_factory, self.sidenav_data_source_factory, self.compiled_host_fs_factory, self.ref_resolver_factory, self.manifest_data_source, svn_constants.PUBLIC_TEMPLATE_PATH, svn_constants.PRIVATE_TEMPLATE_PATH, base_path) self.api_data_source_factory.SetTemplateDataSource( self.template_data_source_factory) self.example_zipper = ExampleZipper(self.compiled_host_fs_factory, self.host_file_system, svn_constants.DOCS_PATH) self.path_canonicalizer = PathCanonicalizer( self.compiled_host_fs_factory) self.redirector = Redirector(self.compiled_host_fs_factory, self.host_file_system, svn_constants.PUBLIC_TEMPLATE_PATH)
def _GetImpl(self): # Cron strategy: # # Find all public template files and static files, and render them. Most of # the time these won't have changed since the last cron run, so it's a # little wasteful, but hopefully rendering is really fast (if it isn't we # have a problem). channel = self._channel logging.info('cron/%s: starting' % channel) # This is returned every time RenderServlet wants to create a new # ServerInstance. server_instance = self._GetSafeServerInstance() def get_via_render_servlet(path): return RenderServlet( Request(path, self._request.host, self._request.headers), _SingletonRenderServletDelegate(server_instance)).Get() def run_cron_for_dir(d, path_prefix=''): success = True start_time = time.time() files = [f for f in server_instance.content_cache.GetFromFileListing(d) if not f.endswith('/')] logging.info('cron/%s: rendering %s files from %s...' % ( channel, len(files), d)) try: for i, f in enumerate(files): error = None path = '%s%s' % (path_prefix, f) try: response = get_via_render_servlet(path) if response.status != 200: error = 'Got %s response' % response.status except DeadlineExceededError: logging.error( 'cron/%s: deadline exceeded rendering %s (%s of %s): %s' % ( channel, path, i + 1, len(files), traceback.format_exc())) raise except error: pass if error: logging.error('cron/%s: error rendering %s: %s' % ( channel, path, error)) success = False finally: logging.info('cron/%s: rendering %s files from %s took %s seconds' % ( channel, len(files), d, time.time() - start_time)) return success success = True try: # Render all of the publicly accessible files. cron_runs = [ # Note: rendering the public templates will pull in all of the private # templates. (svn_constants.PUBLIC_TEMPLATE_PATH, ''), # Note: rendering the public templates will have pulled in the .js # and manifest.json files (for listing examples on the API reference # pages), but there are still images, CSS, etc. (svn_constants.STATIC_PATH, 'static/'), ] if not IsDevServer(): cron_runs.append( (svn_constants.EXAMPLES_PATH, 'extensions/examples/')) # Note: don't try to short circuit any of this stuff. We want to run # the cron for all the directories regardless of intermediate # failures. for path, path_prefix in cron_runs: success = run_cron_for_dir(path, path_prefix=path_prefix) and success # TODO(kalman): Generic way for classes to request cron access. The next # two special cases are ugly. It would potentially greatly speed up cron # runs, too. # Extension examples have zip files too. Well, so do apps, but the app # file system doesn't get the Offline treatment so they don't need cron. if not IsDevServer(): manifest_json = '/manifest.json' example_zips = [ '%s.zip' % filename[:-len(manifest_json)] for filename in server_instance.content_cache.GetFromFileListing( svn_constants.EXAMPLES_PATH) if filename.endswith(manifest_json)] logging.info('cron/%s: rendering %s example zips...' % ( channel, len(example_zips))) start_time = time.time() try: success = success and all( get_via_render_servlet('extensions/examples/%s' % z).status == 200 for z in example_zips) finally: logging.info('cron/%s: rendering %s example zips took %s seconds' % ( channel, len(example_zips), time.time() - start_time)) # Also trigger a redirect so that PathCanonicalizer has an opportunity to # cache file listings. logging.info('cron/%s: triggering a redirect...' % channel) redirect_response = get_via_render_servlet('storage.html') success = success and redirect_response.status == 302 except DeadlineExceededError: success = False logging.info('cron/%s: finished' % channel) return (Response.Ok('Success') if success else Response.InternalError('Failure'))
from fnmatch import fnmatch import logging import mimetypes import os import traceback from appengine_wrappers import IsDevServer from branch_utility import BranchUtility from file_system import FileNotFoundError from server_instance import ServerInstance from servlet import Servlet, Response import svn_constants _DEFAULT_CHANNEL = 'stable' _ALWAYS_ONLINE = IsDevServer() def _IsBinaryMimetype(mimetype): return any( mimetype.startswith(prefix) for prefix in ['audio', 'image', 'video']) def AlwaysOnline(fn): '''A function decorator which forces the rendering to be always online rather than the default offline behaviour. Useful for testing. ''' def impl(*args, **optargs): global _ALWAYS_ONLINE was_always_online = _ALWAYS_ONLINE try:
class Handler(webapp.RequestHandler): # AppEngine instances should never need to call out to SVN. That should only # ever be done by the cronjobs, which then write the result into DataStore, # which is as far as instances look. # # Why? SVN is slow and a bit flaky. Cronjobs failing is annoying but # temporary. Instances failing affects users, and is really bad. # # Anyway - to enforce this, we actually don't give instances access to SVN. # If anything is missing from datastore, it'll be a 404. If the cronjobs # don't manage to catch everything - uhoh. On the other hand, we'll figure it # out pretty soon, and it also means that legitimate 404s are caught before a # round trip to SVN. # # However, we can't expect users of preview.py nor the dev server to run a # cronjob first, so, this is a hack allow that to be online all of the time. # TODO(kalman): achieve this via proper dependency injection. ALWAYS_ONLINE = IsDevServer() def __init__(self, request, response): super(Handler, self).__init__(request, response) def _HandleGet(self, path): channel_name, real_path = BranchUtility.SplitChannelNameFromPath(path) if channel_name == _DEFAULT_CHANNEL: self.redirect('/%s' % real_path) return if channel_name is None: channel_name = _DEFAULT_CHANNEL # TODO(kalman): Check if |path| is a directory and serve path/index.html # rather than special-casing apps/extensions. if real_path.strip('/') == 'apps': real_path = 'apps/index.html' if real_path.strip('/') == 'extensions': real_path = 'extensions/index.html' constructor = (ServerInstance.CreateOnline if Handler.ALWAYS_ONLINE else ServerInstance.GetOrCreateOffline) server_instance = constructor(channel_name) canonical_path = server_instance.path_canonicalizer.Canonicalize( real_path) if real_path != canonical_path: self.redirect(canonical_path) return server_instance.Get(real_path, self.request, self.response) def _HandleCron(self, path): # Cron strategy: # # Find all public template files and static files, and render them. Most of # the time these won't have changed since the last cron run, so it's a # little wasteful, but hopefully rendering is really fast (if it isn't we # have a problem). class MockResponse(object): def __init__(self): self.status = 200 self.out = StringIO() self.headers = {} def set_status(self, status): self.status = status def clear(self, *args): pass class MockRequest(object): def __init__(self, path): self.headers = {} self.path = path self.url = '//localhost/%s' % path channel = path.split('/')[-1] logging.info('cron/%s: starting' % channel) server_instance = ServerInstance.CreateOnline(channel) def run_cron_for_dir(d, path_prefix=''): success = True start_time = time.time() files = [ f for f in server_instance.content_cache.GetFromFileListing(d) if not f.endswith('/') ] logging.info('cron/%s: rendering %s files from %s...' % (channel, len(files), d)) for i, f in enumerate(files): error = None path = '%s%s' % (path_prefix, f) try: response = MockResponse() server_instance.Get(path, MockRequest(path), response) if response.status != 200: error = 'Got %s response' % response.status except DeadlineExceededError: logging.error( 'cron/%s: deadline exceeded rendering %s (%s of %s): %s' % (channel, path, i + 1, len(files), traceback.format_exc())) raise except error: pass if error: logging.error('cron/%s: error rendering %s: %s' % (channel, path, error)) success = False logging.info( 'cron/%s: rendering %s files from %s took %s seconds' % (channel, len(files), d, time.time() - start_time)) return success success = True for path, path_prefix in ( # Note: rendering the public templates will pull in all of the private # templates. (svn_constants.PUBLIC_TEMPLATE_PATH, ''), # Note: rendering the public templates will have pulled in the .js and # manifest.json files (for listing examples on the API reference pages), # but there are still images, CSS, etc. (svn_constants.STATIC_PATH, 'static/'), (svn_constants.EXAMPLES_PATH, 'extensions/examples/')): try: # Note: don't try to short circuit any of this stuff. We want to run # the cron for all the directories regardless of intermediate failures. success = run_cron_for_dir(path, path_prefix=path_prefix) and success except DeadlineExceededError: success = False break if success: self.response.status = 200 self.response.out.write('Success') else: self.response.status = 500 self.response.out.write('Failure') logging.info('cron/%s: finished' % channel) def _RedirectSpecialCases(self, path): google_dev_url = 'http://developer.google.com/chrome' if path == '/' or path == '/index.html': self.redirect(google_dev_url) return True if path == '/apps.html': self.redirect('/apps/about_apps.html') return True return False def _RedirectFromCodeDotGoogleDotCom(self, path): if (not self.request.url.startswith( ('http://code.google.com', 'https://code.google.com'))): return False new_url = 'http://developer.chrome.com/' # switch to https if necessary if (self.request.url.startswith('https')): new_url = new_url.replace('http', 'https', 1) path = path.split('/') if len(path) > 0 and path[0] == 'chrome': path.pop(0) for channel in BranchUtility.GetAllBranchNames(): if channel in path: position = path.index(channel) path.pop(position) path.insert(0, channel) new_url += '/'.join(path) self.redirect(new_url) return True def get(self): path = self.request.path if path in ['favicon.ico', 'robots.txt']: response.set_status(404) return if self._RedirectSpecialCases(path): return if path.startswith('/cron'): # Crons often time out, and when they do *and* then eventually try to # flush logs they die. Turn off autoflush and manually do so at the end. logservice.AUTOFLUSH_ENABLED = False try: self._HandleCron(path) finally: logservice.flush() return # Redirect paths like "directory" to "directory/". This is so relative # file paths will know to treat this as a directory. if os.path.splitext(path)[1] == '' and path[-1] != '/': self.redirect(path + '/') return path = path.strip('/') if self._RedirectFromCodeDotGoogleDotCom(path): return self._HandleGet(path)
def CreateAppSamplesFileSystem(self, object_store_creator): # TODO(kalman): OfflineServerInstance wrapper for GithubFileSystem, but # the cron job doesn't crawl the samples yet. return (EmptyDirFileSystem() if IsDevServer() else GithubFileSystem.Create(object_store_creator))