def _generate_and_email_report( module_versions, recipients, request_id_url, report_url, extras): """Generates and emails an exception report. To be called from a cron_job. Arguments: module_versions: list of tuple of module-version to gather info about. recipients: str containing comma separated email addresses. request_id_url: base url to use to link to a specific request_id. report_url: base url to use to recreate this report. extras: extra dict to use to render the template. Returns: True if the email was sent successfully. """ start_time = _get_default_start_time() end_time = _get_end_time_for_email() logging.info( '_generate_and_email_report(%s, %s, %s, ..., %s)', start_time, end_time, module_versions, recipients) categories, ignored, end_time = logscraper.scrape_logs_for_errors( start_time, end_time, module_versions) if categories: params = _get_template_env(start_time, end_time, module_versions) params.update(extras or {}) params.update( _records_to_params( categories, sum(c.events.total_count for c in ignored), request_id_url, report_url)) body = template.render('ereporter2/email_report_content.html', params) subject_line = template.render( 'ereporter2/email_report_title.html', params) if not _email_html(recipients, subject_line, body): on_error.log( source='server', category='email', message='Failed to email ereporter2 report') logging.info('New timestamp %s', end_time) models.ErrorReportingInfo( key=models.ErrorReportingInfo.primary_key(), timestamp=end_time).put() logging.info( 'Processed %d items, ignored %d, reduced to %d categories, sent to %s.', sum(c.events.total_count for c in categories), sum(c.events.total_count for c in ignored), len(categories), recipients) return True
def render(name, params=None): """Shorthand to render a template.""" out = { 'google_analytics': config.settings().google_analytics, } out.update(params or {}) return template.render(name, out)
def get(self): params = { 'is_admin': auth.is_admin(), } self.response.write( template.render('templates/root.html', params=params))
def get(self): query = self.request.get('q') user = users.get_current_user() data = { 'query': query, 'error': None, 'search_results': [], 'user': { 'email': user.email() if user else None, 'login_url': users.create_login_url(), 'logout_url': users.create_logout_url('/'), } } is_googler = user and user.email().endswith('@google.com') if query: try: data['search_results'] = list( docs.find(query, include_internal=is_googler)) except Exception as ex: logging.exception('Exception during search for "%s"', query) data['error'] = ex.message self.response.set_status(500) self.response.write(template.render('doc/index.html', data))
def _generate_and_email_report(module_versions, recipients, request_id_url, report_url, extras): """Generates and emails an exception report. To be called from a cron_job. Arguments: module_versions: list of tuple of module-version to gather info about. recipients: str containing comma separated email addresses. request_id_url: base url to use to link to a specific request_id. report_url: base url to use to recreate this report. extras: extra dict to use to render the template. Returns: True if the email was sent successfully. """ start_time = _get_default_start_time() end_time = _get_end_time_for_email() logging.info('_generate_and_email_report(%s, %s, %s, ..., %s)', start_time, end_time, module_versions, recipients) categories, ignored, end_time = logscraper.scrape_logs_for_errors( start_time, end_time, module_versions) if categories: params = _get_template_env(start_time, end_time, module_versions) params.update(extras or {}) params.update( _records_to_params(categories, sum(c.events.total_count for c in ignored), request_id_url, report_url)) body = template.render('ereporter2/email_report_content.html', params) subject_line = template.render('ereporter2/email_report_title.html', params) if not _email_html(recipients, subject_line, body): on_error.log(source='server', category='email', message='Failed to email ereporter2 report') logging.info('New timestamp %s', end_time) models.ErrorReportingInfo(key=models.ErrorReportingInfo.primary_key(), timestamp=end_time).put() logging.info( 'Processed %d items, ignored %d, reduced to %d categories, sent to %s.', sum(c.events.total_count for c in categories), sum(c.events.total_count for c in ignored), len(categories), recipients) return True
def get(self, error_id): error = models.Error.get_by_id(int(error_id)) if not error: self.abort(404, 'Error not found') params = { 'error': error, 'now': utils.utcnow(), } self.response.out.write(template.render('ereporter2/error.html', params))
def get(self, error_id): error = models.Error.get_by_id(int(error_id)) if not error: self.abort(404, 'Error not found') params = { 'error': error, 'now': utils.utcnow(), } self.response.out.write( template.render('ereporter2/error.html', params))
def get(self): # Due to historical reasons where created_ts had indexed=False,, do not use # .order(models.ErrorReportingMonitoring.created_ts) yet. Fix this once all # objects have been updated. items = models.ErrorReportingMonitoring.query().fetch() items.sort(key=lambda x: x.created_ts) params = { 'silenced': items, 'xsrf_token': self.generate_xsrf_token(), } self.response.out.write(template.render('ereporter2/silence.html', params))
def get(self): limit = int(self.request.get('limit', 100)) cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor')) errors_found, cursor, more = models.Error.query().order( -models.Error.created_ts).fetch_page(limit, start_cursor=cursor) params = { 'cursor': cursor.urlsafe() if cursor and more else None, 'errors': errors_found, 'limit': limit, 'now': utils.utcnow(), } self.response.out.write(template.render('ereporter2/errors.html', params))
def get(self): # Due to historical reasons where created_ts had indexed=False,, do not use # .order(models.ErrorReportingMonitoring.created_ts) yet. Fix this once all # objects have been updated. items = models.ErrorReportingMonitoring.query().fetch() items.sort(key=lambda x: x.created_ts) params = { 'silenced': items, 'xsrf_token': self.generate_xsrf_token(), } self.response.out.write( template.render('ereporter2/silence.html', params))
def get(self): limit = int(self.request.get('limit', 100)) cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor')) errors_found, cursor, more = models.Error.query().order( -models.Error.created_ts).fetch_page(limit, start_cursor=cursor) params = { 'cursor': cursor.urlsafe() if cursor and more else None, 'errors': errors_found, 'limit': limit, 'now': utils.utcnow(), } self.response.out.write( template.render('ereporter2/errors.html', params))
def get(self): user = users.get_current_user() template_values = { 'user': user, 'xsrf_token': self.generate_xsrf_token(), } latest_msg = BannerMessage.get_last_datastore( BannerMessageHandler.MSG_TYPE) if latest_msg is not None and latest_msg.active: template_values['latest_msg'] = latest_msg self.response.write(template.render('som/banner-msg-form.html', template_values))
def get(self): """Reports the errors logged and ignored. Arguments: start: epoch time to start looking at. Defaults to the messages since the last email. end: epoch time to stop looking at. Defaults to now. modules: comma separated modules to look at. tainted: 0 or 1, specifying if desiring tainted versions. Defaults to 1. """ # TODO(maruel): Be consistent about using either epoch or human readable # formatted datetime. end = int(float(self.request.get('end', 0)) or time.time()) start = int( float(self.request.get('start', 0)) or ui._get_default_start_time() or 0) modules = self.request.get('modules') if modules: modules = modules.split(',') tainted = bool(int(self.request.get('tainted', '1'))) module_versions = utils.get_module_version_list(modules, tainted) errors, ignored, _end_time = logscraper.scrape_logs_for_errors( start, end, module_versions) params = { 'errors': errors, 'errors_count': sum(len(e.events) for e in errors), 'errors_version_count': len(set(itertools.chain.from_iterable(e.versions for e in errors))), 'ignored': ignored, 'ignored_count': sum(len(i.events) for i in ignored), 'ignored_version_count': len(set(itertools.chain.from_iterable(i.versions for i in ignored))), 'xsrf_token': self.generate_xsrf_token(), } params.update(ui._get_template_env(start, end, module_versions)) self.response.write(template.render('ereporter2/requests.html', params))
def get(self, machine_id=None): params = { 'machines': [], 'next_page_token': None, } if machine_id: machine = models.CatalogMachineEntry.get_by_id(machine_id) if not machine: self.abort(404) params['machines'] = [machine] else: query = models.CatalogMachineEntry.query().order( models.CatalogMachineEntry.dimensions.hostname) page_token = self.request.get('page_token') or '' params['machines'], params['next_page_token'] = ( datastore_utils.fetch_page(query, 50, page_token)) self.response.write( template.render('templates/catalog.html', params=params))
def reply(self, path, env=None, status=200): """Render template |path| to response using given environment. Args: path: path to a template, relative to templates/. env: additional environment dict to use when rendering the template. status: HTTP status code to return. """ full_env = { 'app_name': _ui_app_name, 'csp_nonce': self.csp_nonce, 'identity': api.get_current_identity(), 'logout_url': json.dumps(self.create_logout_url('/')), # see base.html 'xsrf_token': self.generate_xsrf_token(), } full_env.update(env or {}) self.response.set_status(status) self.response.headers['Content-Type'] = 'text/html; charset=utf-8' self.response.write(template.render(path, full_env))
def get(self, lease_id=None): params = { 'lease_requests': [], 'next_page_token': None, 'now_ts': utils.time_time(), } if lease_id: lease_request = models.LeaseRequest.get_by_id(lease_id) if not lease_request: self.abort(404) params['lease_requests'] = [lease_request] else: query = models.LeaseRequest.query().order( -models.LeaseRequest.last_modified_ts) page_token = self.request.get('page_token') or '' params['lease_requests'], params['next_page_token'] = ( datastore_utils.fetch_page(query, 50, page_token)) self.response.write(template.render('templates/leases.html', params=params))
def reply(self, path, env=None, status=200): """Render template |path| to response using given environment. Optional keys from |env| that base.html uses: css_file: URL to a file with page specific styles, relative to site root. js_file: URL to a file with page specific Javascript code, relative to site root. File should define global object named same as a filename, i.e. '/auth/static/js/api.js' should define global object 'api' that incapsulates functionality implemented in the module. navbar_tab_id: id a navbar tab to highlight. page_title: title of an HTML page. Args: path: path to a template, relative to templates/. env: additional environment dict to use when rendering the template. status: HTTP status code to return. """ env = (env or {}).copy() env.setdefault("css_file", None) env.setdefault("js_file", None) env.setdefault("navbar_tab_id", None) env.setdefault("page_title", "Untitled") # This goes to both Jinja2 env and Javascript config object. common = { "auth_service_config_locked": False, # overridden in auth_service "login_url": users.create_login_url(self.request.path), "logout_url": users.create_logout_url("/"), "xsrf_token": self.generate_xsrf_token(), } if _ui_env_callback: common.update(_ui_env_callback(self)) # Name of Javascript module with page code. js_module_name = None if env["js_file"]: assert env["js_file"].endswith(".js") js_module_name = os.path.basename(env["js_file"])[:-3] # This will be accessible from Javascript as global 'config' variable. js_config = {"identity": api.get_current_identity().to_bytes()} js_config.update(common) # Prepare URL to explore app API. schema, netloc, _, _, _, _ = urlparse.urlparse(self.request.url) api_url = "https://apis-explorer.appspot.com/apis-explorer/?" "base=%s://%s/_ah/api" % (schema, netloc) # Jinja2 environment to use to render a template. full_env = { "app_name": _ui_app_name, "app_revision_url": utils.get_app_revision_url(), "app_version": utils.get_app_version(), "config": json.dumps(js_config), "identity": api.get_current_identity(), "js_module_name": js_module_name, "api_url": api_url, "navbar": [(cls.navbar_tab_id, cls.navbar_tab_title, cls.navbar_tab_url) for cls in _ui_navbar_tabs], } full_env.update(common) full_env.update(env) # Render it. self.response.set_status(status) self.response.headers["Content-Type"] = "text/html; charset=utf-8" self.response.write(template.render(path, full_env))
def get(self, request_id): data = logscraper._log_request_id(request_id) if not data: self.abort(404, detail='Request id was not found.') self.response.write( template.render('ereporter2/request.html', {'request': data}))
def reply(self, path, env=None, status=200): """Render template |path| to response using given environment. Optional keys from |env| that base.html uses: css_file: URL to a file with page specific styles, relative to site root. js_file: URL to a file with page specific Javascript code, relative to site root. File should define global object named same as a filename, i.e. '/auth/static/js/api.js' should define global object 'api' that incapsulates functionality implemented in the module. navbar_tab_id: id a navbar tab to highlight. page_title: title of an HTML page. Args: path: path to a template, relative to templates/. env: additional environment dict to use when rendering the template. status: HTTP status code to return. """ env = (env or {}).copy() env.setdefault('css_file', None) env.setdefault('js_file', None) env.setdefault('navbar_tab_id', None) env.setdefault('page_title', 'Untitled') # This goes to both Jinja2 env and Javascript config object. common = { 'login_url': users.create_login_url(self.request.path), 'logout_url': users.create_logout_url('/'), 'xsrf_token': self.generate_xsrf_token(), } # Name of Javascript module with page code. js_module_name = None if env['js_file']: assert env['js_file'].endswith('.js') js_module_name = os.path.basename(env['js_file'])[:-3] # This will be accessible from Javascript as global 'config' variable. js_config = { 'identity': api.get_current_identity().to_bytes(), } js_config.update(common) # Jinja2 environment to use to render a template. full_env = { 'app_name': _ui_app_name, 'app_revision_url': utils.get_app_revision_url(), 'app_version': utils.get_app_version(), 'config': json.dumps(js_config), 'identity': api.get_current_identity(), 'js_module_name': js_module_name, 'navbar': [ (cls.navbar_tab_id, cls.navbar_tab_title, cls.navbar_tab_url) for cls in _ui_navbar_tabs ], } full_env.update(common) full_env.update(env) # Render it. self.response.set_status(status) self.response.headers['Content-Type'] = 'text/html; charset=utf-8' self.response.write(template.render(path, full_env))
def notify_gitiles_rejection(config_set, location, validation_result): """Notifies interested parties about an error in a config set revision. Sends a notification per location only once. Args: location (gitiles.Location): an absolute gitiles location of the config set that could not be imported. validation_result (components.config.validation_context.Result). """ assert RE_GIT_HASH.match(location.treeish), location if Notification.get_by_id(str(location)): logging.debug('Notification was already sent.') return log = location.get_log(limit=1) if not log or not log.commits: logging.error('could not load commit %s', location) return commit = log.commits[0] app_id = app_identity.get_application_id() rev = location.treeish[:7] try: template_params = { 'author': commit.author.name or commit.author.email, 'messages': [ { 'severity': logging.getLevelName(msg.severity), 'text': msg.text } for msg in validation_result.messages ], 'rev_link': location, 'rev_hash': rev, 'rev_repo': location.project, 'cur_rev_hash': None, 'cur_rev_link': None, } cs = storage.ConfigSet.get_by_id(config_set) if cs and cs.latest_revision: template_params.update( cur_rev_hash=cs.latest_revision[:7], cur_rev_link=cs.latest_revision_url, ) msg = mail.EmailMessage( sender=( '%s.appspot.com <noreply@%s.appspotmail.com>' % (app_id, app_id)), subject='Config revision %s is rejected' % rev, to=get_recipients(commit), html=template.render( 'templates/validation_notification.html', template_params)) cc = get_cc_recipients() if cc: msg.cc = cc logging.info('Emailing %s', ', '.join(msg.to)) _send(msg) except mail_errors.Error as ex: raise FailedToNotify(ex.message), None, sys.exc_info()[2] Notification(id=str(location)).put()
def reply(self, path, env=None, status=200): """Render template |path| to response using given environment. Optional keys from |env| that base.html uses: css_file: URL to a file with page specific styles, relative to site root. js_file: URL to a file with page specific Javascript code, relative to site root. File should define global object named same as a filename, i.e. '/auth/static/js/api.js' should define global object 'api' that incapsulates functionality implemented in the module. navbar_tab_id: id a navbar tab to highlight. page_title: title of an HTML page. Args: path: path to a template, relative to templates/. env: additional environment dict to use when rendering the template. status: HTTP status code to return. """ env = (env or {}).copy() env.setdefault('css_file', None) env.setdefault('js_file', None) env.setdefault('navbar_tab_id', None) env.setdefault('page_title', 'Untitled') # This goes to both Jinja2 env and Javascript config object. user = self.get_current_user() common = { 'account_picture': user.picture() if user else None, 'auth_service_config_locked': False, # overridden in auth_service 'is_admin': api.is_admin(), 'login_url': self.create_login_url(self.request.url), 'logout_url': self.create_logout_url('/'), 'using_gae_auth': self.auth_method == handler.gae_cookie_authentication, 'xsrf_token': self.generate_xsrf_token(), } if _ui_env_callback: common.update(_ui_env_callback(self)) # Name of Javascript module with page code. js_module_name = None if env['js_file']: assert env['js_file'].endswith('.js') js_module_name = os.path.basename(env['js_file'])[:-3] # This will be accessible from Javascript as global 'config' variable. js_config = { 'identity': api.get_current_identity().to_bytes(), } js_config.update(common) # Prepare URL to explore app API. schema, netloc = urlparse.urlparse(self.request.url)[:2] api_url = ( 'https://apis-explorer.appspot.com/apis-explorer/?' 'base=%s://%s/_ah/api' % (schema, netloc)) # Jinja2 environment to use to render a template. full_env = { 'app_name': _ui_app_name, 'app_revision_url': utils.get_app_revision_url(), 'app_version': utils.get_app_version(), 'config': json.dumps(js_config), 'identity': api.get_current_identity(), 'js_module_name': js_module_name, 'api_url': api_url, 'navbar': [ (cls.navbar_tab_id, cls.navbar_tab_title, cls.navbar_tab_url) for cls in _ui_navbar_tabs if cls.is_visible() ], } full_env.update(common) full_env.update(env) # Render it. self.response.set_status(status) self.response.headers['Content-Type'] = 'text/html; charset=utf-8' self.response.write(template.render(path, full_env))
def get(self): self.response.write(template.render( 'adapter/proxy.html', params={'base_path': base_path}))
def render(name, params=None): """Shorthand to render a template.""" return template.render(name, params)
def reply(self, path, env=None, status=200): """Renders template |path| to the HTTP response using given environment. Optional keys from |env| that base.html uses: css_file: URL to a file with page specific styles, relative to site root. js_file: URL to a file with page specific Javascript code, relative to site root. File should define global object named same as a filename, i.e. '/auth/static/js/api.js' should define global object 'api' that incapsulates functionality implemented in the module. navbar_tab_id: id of a navbar tab to highlight. page_title: title of an HTML page. Args: path: path to a template, relative to templates/. env: additional environment dict to use when rendering the template. status: HTTP status code to return. """ env = (env or {}).copy() env.setdefault('css_file', None) env.setdefault('js_file', None) env.setdefault('navbar_tab_id', None) env.setdefault('page_title', 'Untitled') # This goes to both Jinja2 env and Javascript config object. user = self.get_current_user() common = { 'account_picture': user.picture() if user else None, 'auth_service_config_locked': False, # overridden in auth_service 'is_admin': api.is_admin(), 'login_url': self.create_login_url(self.request.url), 'logout_url': self.create_logout_url('/'), 'using_gae_auth': self.auth_method == handler.gae_cookie_authentication, 'xsrf_token': self.generate_xsrf_token(), } if _ui_data_callback: common.update(_ui_data_callback()) # Name of Javascript module with page code. js_module_name = None if env['js_file']: assert env['js_file'].endswith('.js') js_module_name = os.path.basename(env['js_file'])[:-3] # This will be accessible from Javascript as global 'config' variable. js_config = { 'identity': api.get_current_identity().to_bytes(), } js_config.update(common) # Jinja2 environment to use to render a template. full_env = { 'app_name': _ui_app_name, 'app_version': utils.get_app_version(), 'config': json.dumps(js_config), 'csp_nonce': self.csp_nonce, 'identity': api.get_current_identity(), 'js_module_name': js_module_name, 'navbar': [ (cls.navbar_tab_id, cls.navbar_tab_title, cls.navbar_tab_url) for cls in _ui_navbar_tabs if cls.is_visible() ], } full_env.update(common) full_env.update(env) # Render it. self.response.set_status(status) self.response.headers['Content-Type'] = 'text/html; charset=utf-8' self.response.write(template.render(path, full_env))
def notify_gitiles_rejection(config_set, location, validation_result): """Notifies interested parties about an error in a config set revision. Sends a notification per location only once. Args: location (gitiles.Location): an absolute gitiles location that could not be imported. validation_result (components.config.validation_context.Result). """ assert RE_GIT_HASH.match(location.treeish), location if Notification.get_by_id(str(location)): logging.debug('Notification was already sent.') return log = location.get_log(limit=1) if not log or not log.commits: logging.error('could not load commit %s', location) return commit = log.commits[0] app_id = app_identity.get_application_id() rev = location.treeish[:7] try: template_params = { 'author': commit.author.name or commit.author.email, 'messages': [ { 'severity': logging.getLevelName(msg.severity), 'text': msg.text } for msg in validation_result.messages ], 'rev_link': location, 'rev_hash': rev, 'rev_repo': location.project, 'cur_rev_hash': None, 'cur_rev_link': None, } cs = storage.ConfigSet.get_by_id(config_set) if cs: template_params.update( cur_rev_hash=cs.latest_revision[:7], cur_rev_link=location._replace(treeish=cs.latest_revision), ) msg = mail.EmailMessage( sender=( '%s.appspot.com <*****@*****.**>' % (app_id, app_id)), subject='Config revision %s is rejected' % rev, to=get_recipients(commit), html=template.render( 'templates/validation_notification.html', template_params)) cc = get_cc_recipients() if cc: msg.cc = cc logging.info('Emailing %s', ', '.join(msg.to)) _send(msg) except mail_errors.Error as ex: raise FailedToNotify(ex.message), None, sys.exc_info()[2] Notification(id=str(location)).put()