def _CreateDataSources(self, request): server_instance = self._server_instance data_sources = CreateDataSources(server_instance, request=request) data_sources.update({ 'samples': server_instance.samples_data_source_factory.Create(request), }) return data_sources
def _GetImpl(self): # Cron strategy: # # Collect all DataSources, the PlatformBundle, the ContentProviders, and # any other statically renderered contents (e.g. examples content), # and spin up taskqueue tasks which will refresh any cached data relevant # to these assets. # # TODO(rockot/kalman): At the moment examples are not actually refreshed # because they're too slow. _log.info('starting') server_instance = self._GetSafeServerInstance() master_fs = server_instance.host_file_system_provider.GetMaster() master_commit = master_fs.GetCommitID().Get() # This is the guy that would be responsible for refreshing the cache of # examples. Here for posterity, hopefully it will be added to the targets # below someday. render_refresher = RenderRefresher(server_instance, self._request) # Get the default taskqueue queue = taskqueue.Queue() # GAE documentation specifies that it's bad to add tasks to a queue # within one second of purging. We wait 2 seconds, because we like # to go the extra mile. queue.purge() time.sleep(2) success = True try: data_sources = CreateDataSources(server_instance) targets = ( data_sources.items() + [('content_providers', server_instance.content_providers), ('platform_bundle', server_instance.platform_bundle)]) title = 'initializing %s parallel targets' % len(targets) _log.info(title) timer = Timer() for name, target in targets: refresh_paths = target.GetRefreshPaths() for path in refresh_paths: queue.add( taskqueue.Task(url='/_refresh/%s/%s' % (name, path), params={'commit': master_commit})) _log.info('%s took %s' % (title, timer.Stop().FormatElapsed())) except: # This should never actually happen (each cron step does its own # conservative error checking), so re-raise no matter what it is. _log.error('uncaught error: %s' % traceback.format_exc()) success = False raise finally: _log.info('finished (%s)', 'success' if success else 'FAILED') return (Response.Ok('Success') if success else Response.InternalError('Failure'))
def _GetImpl(self): # Cron strategy: # # Collect all DataSources, the PlatformBundle, the ContentProviders, and # any other statically renderered contents (e.g. examples content), # and spin up taskqueue tasks which will refresh any cached data relevant # to these assets. # # TODO(rockot/kalman): At the moment examples are not actually refreshed # because they're too slow. _log.info('starting') server_instance = self._GetSafeServerInstance() master_fs = server_instance.host_file_system_provider.GetMaster() master_commit = master_fs.GetCommitID().Get() # This is the guy that would be responsible for refreshing the cache of # examples. Here for posterity, hopefully it will be added to the targets # below someday. render_refresher = RenderRefresher(server_instance, self._request) # Get the default taskqueue queue = taskqueue.Queue() # GAE documentation specifies that it's bad to add tasks to a queue # within one second of purging. We wait 2 seconds, because we like # to go the extra mile. queue.purge() time.sleep(2) success = True try: data_sources = CreateDataSources(server_instance) targets = (data_sources.items() + [('content_providers', server_instance.content_providers), ('platform_bundle', server_instance.platform_bundle)]) title = 'initializing %s parallel targets' % len(targets) _log.info(title) timer = Timer() for name, target in targets: refresh_paths = target.GetRefreshPaths() for path in refresh_paths: queue.add(taskqueue.Task(url='/_refresh/%s/%s' % (name, path), params={'commit': master_commit})) _log.info('%s took %s' % (title, timer.Stop().FormatElapsed())) except: # This should never actually happen (each cron step does its own # conservative error checking), so re-raise no matter what it is. _log.error('uncaught error: %s' % traceback.format_exc()) success = False raise finally: _log.info('finished (%s)', 'success' if success else 'FAILED') return (Response.Ok('Success') if success else Response.InternalError('Failure'))
def Render(self, template, request, data_sources=None, additional_context=None): '''Renders |template| using |request|. Specify |data_sources| to only include the DataSources with the given names when rendering the template. Specify |additional_context| to inject additional template context when rendering the template. ''' assert isinstance(template, Handlebar), type(template) render_context = CreateDataSources(self._server_instance, request) if data_sources is not None: render_context = dict((name, d) for name, d in render_context.iteritems() if name in data_sources) render_context.update({ 'apps_samples_url': GITHUB_BASE, 'base_path': self._server_instance.base_path, 'extensions_samples_url': EXTENSIONS_SAMPLES, 'static': self._server_instance.base_path + 'static', }) render_context.update(additional_context or {}) render_data = template.Render(render_context) return render_data.text, render_data.errors
def UpdateCache(single_data_source=None, commit=None): '''Attempts to populate the datastore with a bunch of information derived from a given commit. ''' server_instance = _CreateServerInstance(commit) # This is the guy that would be responsible for refreshing the cache of # examples. Here for posterity, hopefully it will be added to the targets # below someday. # render_refresher = RenderRefresher(server_instance, self._request) data_sources = CreateDataSources(server_instance) data_sources['content_providers'] = server_instance.content_providers data_sources['platform_bundle'] = server_instance.platform_bundle if single_data_source: _UpdateDataSource(single_data_source, data_sources[single_data_source]) else: for name, source in data_sources.iteritems(): _UpdateDataSource(name, source)
def Render(self, template, request, data_sources=None, additional_context=None): '''Renders |template| using |request|. Specify |data_sources| to only include the DataSources with the given names when rendering the template. Specify |additional_context| to inject additional template context when rendering the template. ''' assert isinstance(template, Motemplate), type(template) render_context = CreateDataSources(self._server_instance, request) if data_sources is not None: render_context = dict((name, d) for name, d in render_context.iteritems() if name in data_sources) render_context.update({ 'apps_samples_url': GITHUB_BASE, 'base_path': self._server_instance.base_path, 'extensions_samples_url': EXTENSIONS_SAMPLES, 'static': self._server_instance.base_path + 'static', }) render_context.update(additional_context or {}) render_data = template.Render(render_context) return render_data.text, render_data.errors
def Render(self, template, request): assert isinstance(template, Handlebar), type(template) server_instance = self._server_instance render_context = { 'api_list': server_instance.api_list_data_source_factory.Create(), 'apis': server_instance.api_data_source_factory.Create(request), 'apps_samples_url': GITHUB_BASE, 'base_path': server_instance.base_path, 'extensions_samples_url': EXTENSIONS_SAMPLES, 'false': False, 'intros': server_instance.intro_data_source_factory.Create(), 'samples': server_instance.samples_data_source_factory.Create(request), 'static': server_instance.base_path + 'static', 'true': True, } render_context.update(CreateDataSources(server_instance, request=request)) render_data = template.render(render_context) if render_data.errors: logging.error('Handlebar error(s) rendering %s:\n%s' % (template._name, ' \n'.join(render_data.errors))) return render_data.text
def _GetImpl(self): # Cron strategy: # # Find all public template files and static files, and render them. Most of # the time these won't have changed since the last cron run, so it's a # little wasteful, but hopefully rendering is really fast (if it isn't we # have a problem). _cronlog.info('starting') # This is returned every time RenderServlet wants to create a new # ServerInstance. # # TODO(kalman): IMPORTANT. This sometimes throws an exception, breaking # everything. Need retry logic at the fetcher level. server_instance = self._GetSafeServerInstance() trunk_fs = server_instance.host_file_system_provider.GetTrunk() def render(path): request = Request(path, self._request.host, self._request.headers) delegate = _SingletonRenderServletDelegate(server_instance) return RenderServlet(request, delegate).Get() def request_files_in_dir(path, prefix='', strip_ext=None): '''Requests every file found under |path| in this host file system, with a request prefix of |prefix|. |strip_ext| is an optional list of file extensions that should be stripped from paths before requesting. ''' def maybe_strip_ext(name): if name == SITE_VERIFICATION_FILE or not strip_ext: return name base, ext = posixpath.splitext(name) return base if ext in strip_ext else name files = [ maybe_strip_ext(name) for name, _ in CreateURLsFromPaths(trunk_fs, path, prefix) ] return _RequestEachItem(path, files, render) results = [] try: # Start running the hand-written Cron methods first; they can be run in # parallel. They are resolved at the end. def run_cron_for_future(target): title = target.__class__.__name__ future, init_timer = TimerClosure(target.Cron) assert isinstance( future, Future), ('%s.Cron() did not return a Future' % title) def resolve(): resolve_timer = Timer() try: future.Get() except Exception as e: _cronlog.error('%s: error %s' % (title, traceback.format_exc())) results.append(False) if IsDeadlineExceededError(e): raise finally: resolve_timer.Stop() _cronlog.info( '%s took %s: %s to initialize and %s to resolve' % (title, init_timer.With(resolve_timer).FormatElapsed(), init_timer.FormatElapsed(), resolve_timer.FormatElapsed())) return Future(delegate=Gettable(resolve)) targets = (CreateDataSources(server_instance).values() + [server_instance.content_providers]) title = 'initializing %s parallel Cron targets' % len(targets) _cronlog.info(title) timer = Timer() try: cron_futures = [ run_cron_for_future(target) for target in targets ] finally: _cronlog.info('%s took %s' % (title, timer.Stop().FormatElapsed())) # Rendering the public templates will also pull in all of the private # templates. results.append( request_files_in_dir(PUBLIC_TEMPLATES, strip_ext=('.html', '.md'))) # Rendering the public templates will have pulled in the .js and # manifest.json files (for listing examples on the API reference pages), # but there are still images, CSS, etc. results.append(request_files_in_dir(STATIC_DOCS, prefix='static')) # Samples are too expensive to run on the dev server, where there is no # parallel fetch. if not IsDevServer(): # Fetch each individual sample file. results.append( request_files_in_dir(EXAMPLES, prefix='extensions/examples')) # Fetch the zip file of each example (contains all the individual # files). example_zips = [] for root, _, files in trunk_fs.Walk(EXAMPLES): example_zips.extend(root + '.zip' for name in files if name == 'manifest.json') results.append( _RequestEachItem( 'example zips', example_zips, lambda path: render('extensions/examples/' + path))) # Resolve the hand-written Cron method futures. title = 'resolving %s parallel Cron targets' % len(targets) _cronlog.info(title) timer = Timer() try: for future in cron_futures: future.Get() finally: _cronlog.info('%s took %s' % (title, timer.Stop().FormatElapsed())) except: results.append(False) # This should never actually happen (each cron step does its own # conservative error checking), so re-raise no matter what it is. _cronlog.error('uncaught error: %s' % traceback.format_exc()) raise finally: success = all(results) _cronlog.info('finished (%s)', 'success' if success else 'FAILED') return (Response.Ok('Success') if success else Response.InternalError('Failure'))