def _generate_and_email_report(
module_versions, recipients, request_id_url, report_url, extras):
"""Generates and emails an exception report.
To be called from a cron_job.
Arguments:
module_versions: list of tuple of module-version to gather info about.
recipients: str containing comma separated email addresses.
request_id_url: base url to use to link to a specific request_id.
report_url: base url to use to recreate this report.
extras: extra dict to use to render the template.
Returns:
True if the email was sent successfully.
"""
start_time = _get_default_start_time()
end_time = _get_end_time_for_email()
logging.info(
'_generate_and_email_report(%s, %s, %s, ..., %s)',
start_time, end_time, module_versions, recipients)
categories, ignored, end_time = logscraper.scrape_logs_for_errors(
start_time, end_time, module_versions)
if categories:
params = _get_template_env(start_time, end_time, module_versions)
params.update(extras or {})
params.update(
_records_to_params(
categories, sum(c.events.total_count for c in ignored),
request_id_url, report_url))
body = template.render('ereporter2/email_report_content.html', params)
subject_line = template.render(
'ereporter2/email_report_title.html', params)
if not _email_html(recipients, subject_line, body):
on_error.log(
source='server',
category='email',
message='Failed to email ereporter2 report')
logging.info('New timestamp %s', end_time)
models.ErrorReportingInfo(
key=models.ErrorReportingInfo.primary_key(),
timestamp=end_time).put()
logging.info(
'Processed %d items, ignored %d, reduced to %d categories, sent to %s.',
sum(c.events.total_count for c in categories),
sum(c.events.total_count for c in ignored),
len(categories),
recipients)
return True
def render(name, params=None):
"""Shorthand to render a template."""
out = {
'google_analytics': config.settings().google_analytics,
}
out.update(params or {})
return template.render(name, out)
def render(name, params=None):
"""Shorthand to render a template."""
out = {
'google_analytics': config.settings().google_analytics,
}
out.update(params or {})
return template.render(name, out)
def get(self):
params = {
'is_admin': auth.is_admin(),
}
self.response.write(
template.render('templates/root.html', params=params))
def get(self):
query = self.request.get('q')
user = users.get_current_user()
data = {
'query': query,
'error': None,
'search_results': [],
'user': {
'email': user.email() if user else None,
'login_url': users.create_login_url(),
'logout_url': users.create_logout_url('/'),
}
}
is_googler = user and user.email().endswith('@google.com')
if query:
try:
data['search_results'] = list(
docs.find(query, include_internal=is_googler))
except Exception as ex:
logging.exception('Exception during search for "%s"', query)
data['error'] = ex.message
self.response.set_status(500)
self.response.write(template.render('doc/index.html', data))
def _generate_and_email_report(module_versions, recipients, request_id_url,
report_url, extras):
"""Generates and emails an exception report.
To be called from a cron_job.
Arguments:
module_versions: list of tuple of module-version to gather info about.
recipients: str containing comma separated email addresses.
request_id_url: base url to use to link to a specific request_id.
report_url: base url to use to recreate this report.
extras: extra dict to use to render the template.
Returns:
True if the email was sent successfully.
"""
start_time = _get_default_start_time()
end_time = _get_end_time_for_email()
logging.info('_generate_and_email_report(%s, %s, %s, ..., %s)', start_time,
end_time, module_versions, recipients)
categories, ignored, end_time = logscraper.scrape_logs_for_errors(
start_time, end_time, module_versions)
if categories:
params = _get_template_env(start_time, end_time, module_versions)
params.update(extras or {})
params.update(
_records_to_params(categories,
sum(c.events.total_count for c in ignored),
request_id_url, report_url))
body = template.render('ereporter2/email_report_content.html', params)
subject_line = template.render('ereporter2/email_report_title.html',
params)
if not _email_html(recipients, subject_line, body):
on_error.log(source='server',
category='email',
message='Failed to email ereporter2 report')
logging.info('New timestamp %s', end_time)
models.ErrorReportingInfo(key=models.ErrorReportingInfo.primary_key(),
timestamp=end_time).put()
logging.info(
'Processed %d items, ignored %d, reduced to %d categories, sent to %s.',
sum(c.events.total_count for c in categories),
sum(c.events.total_count for c in ignored), len(categories),
recipients)
return True
def get(self, error_id):
error = models.Error.get_by_id(int(error_id))
if not error:
self.abort(404, 'Error not found')
params = {
'error': error,
'now': utils.utcnow(),
}
self.response.out.write(template.render('ereporter2/error.html', params))
def get(self, error_id):
error = models.Error.get_by_id(int(error_id))
if not error:
self.abort(404, 'Error not found')
params = {
'error': error,
'now': utils.utcnow(),
}
self.response.out.write(
template.render('ereporter2/error.html', params))
def get(self):
# Due to historical reasons where created_ts had indexed=False,, do not use
# .order(models.ErrorReportingMonitoring.created_ts) yet. Fix this once all
# objects have been updated.
items = models.ErrorReportingMonitoring.query().fetch()
items.sort(key=lambda x: x.created_ts)
params = {
'silenced': items,
'xsrf_token': self.generate_xsrf_token(),
}
self.response.out.write(template.render('ereporter2/silence.html', params))
def get(self):
limit = int(self.request.get('limit', 100))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
errors_found, cursor, more = models.Error.query().order(
-models.Error.created_ts).fetch_page(limit, start_cursor=cursor)
params = {
'cursor': cursor.urlsafe() if cursor and more else None,
'errors': errors_found,
'limit': limit,
'now': utils.utcnow(),
}
self.response.out.write(template.render('ereporter2/errors.html', params))
def get(self):
# Due to historical reasons where created_ts had indexed=False,, do not use
# .order(models.ErrorReportingMonitoring.created_ts) yet. Fix this once all
# objects have been updated.
items = models.ErrorReportingMonitoring.query().fetch()
items.sort(key=lambda x: x.created_ts)
params = {
'silenced': items,
'xsrf_token': self.generate_xsrf_token(),
}
self.response.out.write(
template.render('ereporter2/silence.html', params))
def get(self):
limit = int(self.request.get('limit', 100))
cursor = datastore_query.Cursor(urlsafe=self.request.get('cursor'))
errors_found, cursor, more = models.Error.query().order(
-models.Error.created_ts).fetch_page(limit, start_cursor=cursor)
params = {
'cursor': cursor.urlsafe() if cursor and more else None,
'errors': errors_found,
'limit': limit,
'now': utils.utcnow(),
}
self.response.out.write(
template.render('ereporter2/errors.html', params))
def get(self):
user = users.get_current_user()
template_values = {
'user': user,
'xsrf_token': self.generate_xsrf_token(),
}
latest_msg = BannerMessage.get_last_datastore(
BannerMessageHandler.MSG_TYPE)
if latest_msg is not None and latest_msg.active:
template_values['latest_msg'] = latest_msg
self.response.write(template.render('som/banner-msg-form.html',
template_values))
def get(self):
"""Reports the errors logged and ignored.
Arguments:
start: epoch time to start looking at. Defaults to the messages since the
last email.
end: epoch time to stop looking at. Defaults to now.
modules: comma separated modules to look at.
tainted: 0 or 1, specifying if desiring tainted versions. Defaults to 1.
"""
# TODO(maruel): Be consistent about using either epoch or human readable
# formatted datetime.
end = int(float(self.request.get('end', 0)) or time.time())
start = int(
float(self.request.get('start', 0))
or ui._get_default_start_time() or 0)
modules = self.request.get('modules')
if modules:
modules = modules.split(',')
tainted = bool(int(self.request.get('tainted', '1')))
module_versions = utils.get_module_version_list(modules, tainted)
errors, ignored, _end_time = logscraper.scrape_logs_for_errors(
start, end, module_versions)
params = {
'errors':
errors,
'errors_count':
sum(len(e.events) for e in errors),
'errors_version_count':
len(set(itertools.chain.from_iterable(e.versions
for e in errors))),
'ignored':
ignored,
'ignored_count':
sum(len(i.events) for i in ignored),
'ignored_version_count':
len(set(itertools.chain.from_iterable(i.versions
for i in ignored))),
'xsrf_token':
self.generate_xsrf_token(),
}
params.update(ui._get_template_env(start, end, module_versions))
self.response.write(template.render('ereporter2/requests.html',
params))
def get(self, machine_id=None):
params = {
'machines': [],
'next_page_token': None,
}
if machine_id:
machine = models.CatalogMachineEntry.get_by_id(machine_id)
if not machine:
self.abort(404)
params['machines'] = [machine]
else:
query = models.CatalogMachineEntry.query().order(
models.CatalogMachineEntry.dimensions.hostname)
page_token = self.request.get('page_token') or ''
params['machines'], params['next_page_token'] = (
datastore_utils.fetch_page(query, 50, page_token))
self.response.write(
template.render('templates/catalog.html', params=params))
def reply(self, path, env=None, status=200):
"""Render template |path| to response using given environment.
Args:
path: path to a template, relative to templates/.
env: additional environment dict to use when rendering the template.
status: HTTP status code to return.
"""
full_env = {
'app_name': _ui_app_name,
'csp_nonce': self.csp_nonce,
'identity': api.get_current_identity(),
'logout_url': json.dumps(self.create_logout_url('/')), # see base.html
'xsrf_token': self.generate_xsrf_token(),
}
full_env.update(env or {})
self.response.set_status(status)
self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
self.response.write(template.render(path, full_env))
def get(self, lease_id=None):
params = {
'lease_requests': [],
'next_page_token': None,
'now_ts': utils.time_time(),
}
if lease_id:
lease_request = models.LeaseRequest.get_by_id(lease_id)
if not lease_request:
self.abort(404)
params['lease_requests'] = [lease_request]
else:
query = models.LeaseRequest.query().order(
-models.LeaseRequest.last_modified_ts)
page_token = self.request.get('page_token') or ''
params['lease_requests'], params['next_page_token'] = (
datastore_utils.fetch_page(query, 50, page_token))
self.response.write(template.render('templates/leases.html', params=params))
def get(self):
"""Reports the errors logged and ignored.
Arguments:
start: epoch time to start looking at. Defaults to the messages since the
last email.
end: epoch time to stop looking at. Defaults to now.
modules: comma separated modules to look at.
tainted: 0 or 1, specifying if desiring tainted versions. Defaults to 1.
"""
# TODO(maruel): Be consistent about using either epoch or human readable
# formatted datetime.
end = int(float(self.request.get('end', 0)) or time.time())
start = int(
float(self.request.get('start', 0)) or
ui._get_default_start_time() or 0)
modules = self.request.get('modules')
if modules:
modules = modules.split(',')
tainted = bool(int(self.request.get('tainted', '1')))
module_versions = utils.get_module_version_list(modules, tainted)
errors, ignored, _end_time = logscraper.scrape_logs_for_errors(
start, end, module_versions)
params = {
'errors': errors,
'errors_count': sum(len(e.events) for e in errors),
'errors_version_count':
len(set(itertools.chain.from_iterable(e.versions for e in errors))),
'ignored': ignored,
'ignored_count': sum(len(i.events) for i in ignored),
'ignored_version_count':
len(set(itertools.chain.from_iterable(i.versions for i in ignored))),
'xsrf_token': self.generate_xsrf_token(),
}
params.update(ui._get_template_env(start, end, module_versions))
self.response.write(template.render('ereporter2/requests.html', params))
def reply(self, path, env=None, status=200):
"""Render template |path| to response using given environment.
Optional keys from |env| that base.html uses:
css_file: URL to a file with page specific styles, relative to site root.
js_file: URL to a file with page specific Javascript code, relative to
site root. File should define global object named same as a filename,
i.e. '/auth/static/js/api.js' should define global object 'api' that
incapsulates functionality implemented in the module.
navbar_tab_id: id a navbar tab to highlight.
page_title: title of an HTML page.
Args:
path: path to a template, relative to templates/.
env: additional environment dict to use when rendering the template.
status: HTTP status code to return.
"""
env = (env or {}).copy()
env.setdefault("css_file", None)
env.setdefault("js_file", None)
env.setdefault("navbar_tab_id", None)
env.setdefault("page_title", "Untitled")
# This goes to both Jinja2 env and Javascript config object.
common = {
"auth_service_config_locked": False, # overridden in auth_service
"login_url": users.create_login_url(self.request.path),
"logout_url": users.create_logout_url("/"),
"xsrf_token": self.generate_xsrf_token(),
}
if _ui_env_callback:
common.update(_ui_env_callback(self))
# Name of Javascript module with page code.
js_module_name = None
if env["js_file"]:
assert env["js_file"].endswith(".js")
js_module_name = os.path.basename(env["js_file"])[:-3]
# This will be accessible from Javascript as global 'config' variable.
js_config = {"identity": api.get_current_identity().to_bytes()}
js_config.update(common)
# Prepare URL to explore app API.
schema, netloc, _, _, _, _ = urlparse.urlparse(self.request.url)
api_url = "https://apis-explorer.appspot.com/apis-explorer/?" "base=%s://%s/_ah/api" % (schema, netloc)
# Jinja2 environment to use to render a template.
full_env = {
"app_name": _ui_app_name,
"app_revision_url": utils.get_app_revision_url(),
"app_version": utils.get_app_version(),
"config": json.dumps(js_config),
"identity": api.get_current_identity(),
"js_module_name": js_module_name,
"api_url": api_url,
"navbar": [(cls.navbar_tab_id, cls.navbar_tab_title, cls.navbar_tab_url) for cls in _ui_navbar_tabs],
}
full_env.update(common)
full_env.update(env)
# Render it.
self.response.set_status(status)
self.response.headers["Content-Type"] = "text/html; charset=utf-8"
self.response.write(template.render(path, full_env))
def get(self, request_id):
data = logscraper._log_request_id(request_id)
if not data:
self.abort(404, detail='Request id was not found.')
self.response.write(
template.render('ereporter2/request.html', {'request': data}))
def reply(self, path, env=None, status=200):
"""Render template |path| to response using given environment.
Optional keys from |env| that base.html uses:
css_file: URL to a file with page specific styles, relative to site root.
js_file: URL to a file with page specific Javascript code, relative to
site root. File should define global object named same as a filename,
i.e. '/auth/static/js/api.js' should define global object 'api' that
incapsulates functionality implemented in the module.
navbar_tab_id: id a navbar tab to highlight.
page_title: title of an HTML page.
Args:
path: path to a template, relative to templates/.
env: additional environment dict to use when rendering the template.
status: HTTP status code to return.
"""
env = (env or {}).copy()
env.setdefault('css_file', None)
env.setdefault('js_file', None)
env.setdefault('navbar_tab_id', None)
env.setdefault('page_title', 'Untitled')
# This goes to both Jinja2 env and Javascript config object.
common = {
'login_url': users.create_login_url(self.request.path),
'logout_url': users.create_logout_url('/'),
'xsrf_token': self.generate_xsrf_token(),
}
# Name of Javascript module with page code.
js_module_name = None
if env['js_file']:
assert env['js_file'].endswith('.js')
js_module_name = os.path.basename(env['js_file'])[:-3]
# This will be accessible from Javascript as global 'config' variable.
js_config = {
'identity': api.get_current_identity().to_bytes(),
}
js_config.update(common)
# Jinja2 environment to use to render a template.
full_env = {
'app_name': _ui_app_name,
'app_revision_url': utils.get_app_revision_url(),
'app_version': utils.get_app_version(),
'config': json.dumps(js_config),
'identity': api.get_current_identity(),
'js_module_name': js_module_name,
'navbar': [
(cls.navbar_tab_id, cls.navbar_tab_title, cls.navbar_tab_url)
for cls in _ui_navbar_tabs
],
}
full_env.update(common)
full_env.update(env)
# Render it.
self.response.set_status(status)
self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
self.response.write(template.render(path, full_env))
def notify_gitiles_rejection(config_set, location, validation_result):
"""Notifies interested parties about an error in a config set revision.
Sends a notification per location only once.
Args:
location (gitiles.Location): an absolute gitiles location of the config set
that could not be imported.
validation_result (components.config.validation_context.Result).
"""
assert RE_GIT_HASH.match(location.treeish), location
if Notification.get_by_id(str(location)):
logging.debug('Notification was already sent.')
return
log = location.get_log(limit=1)
if not log or not log.commits:
logging.error('could not load commit %s', location)
return
commit = log.commits[0]
app_id = app_identity.get_application_id()
rev = location.treeish[:7]
try:
template_params = {
'author': commit.author.name or commit.author.email,
'messages': [
{
'severity': logging.getLevelName(msg.severity),
'text': msg.text
}
for msg in validation_result.messages
],
'rev_link': location,
'rev_hash': rev,
'rev_repo': location.project,
'cur_rev_hash': None,
'cur_rev_link': None,
}
cs = storage.ConfigSet.get_by_id(config_set)
if cs and cs.latest_revision:
template_params.update(
cur_rev_hash=cs.latest_revision[:7],
cur_rev_link=cs.latest_revision_url,
)
msg = mail.EmailMessage(
sender=(
'%s.appspot.com <noreply@%s.appspotmail.com>' % (app_id, app_id)),
subject='Config revision %s is rejected' % rev,
to=get_recipients(commit),
html=template.render(
'templates/validation_notification.html', template_params))
cc = get_cc_recipients()
if cc:
msg.cc = cc
logging.info('Emailing %s', ', '.join(msg.to))
_send(msg)
except mail_errors.Error as ex:
raise FailedToNotify(ex.message), None, sys.exc_info()[2]
Notification(id=str(location)).put()
def reply(self, path, env=None, status=200):
"""Render template |path| to response using given environment.
Optional keys from |env| that base.html uses:
css_file: URL to a file with page specific styles, relative to site root.
js_file: URL to a file with page specific Javascript code, relative to
site root. File should define global object named same as a filename,
i.e. '/auth/static/js/api.js' should define global object 'api' that
incapsulates functionality implemented in the module.
navbar_tab_id: id a navbar tab to highlight.
page_title: title of an HTML page.
Args:
path: path to a template, relative to templates/.
env: additional environment dict to use when rendering the template.
status: HTTP status code to return.
"""
env = (env or {}).copy()
env.setdefault('css_file', None)
env.setdefault('js_file', None)
env.setdefault('navbar_tab_id', None)
env.setdefault('page_title', 'Untitled')
# This goes to both Jinja2 env and Javascript config object.
user = self.get_current_user()
common = {
'account_picture': user.picture() if user else None,
'auth_service_config_locked': False, # overridden in auth_service
'is_admin': api.is_admin(),
'login_url': self.create_login_url(self.request.url),
'logout_url': self.create_logout_url('/'),
'using_gae_auth': self.auth_method == handler.gae_cookie_authentication,
'xsrf_token': self.generate_xsrf_token(),
}
if _ui_env_callback:
common.update(_ui_env_callback(self))
# Name of Javascript module with page code.
js_module_name = None
if env['js_file']:
assert env['js_file'].endswith('.js')
js_module_name = os.path.basename(env['js_file'])[:-3]
# This will be accessible from Javascript as global 'config' variable.
js_config = {
'identity': api.get_current_identity().to_bytes(),
}
js_config.update(common)
# Prepare URL to explore app API.
schema, netloc = urlparse.urlparse(self.request.url)[:2]
api_url = (
'https://apis-explorer.appspot.com/apis-explorer/?'
'base=%s://%s/_ah/api' % (schema, netloc))
# Jinja2 environment to use to render a template.
full_env = {
'app_name': _ui_app_name,
'app_revision_url': utils.get_app_revision_url(),
'app_version': utils.get_app_version(),
'config': json.dumps(js_config),
'identity': api.get_current_identity(),
'js_module_name': js_module_name,
'api_url': api_url,
'navbar': [
(cls.navbar_tab_id, cls.navbar_tab_title, cls.navbar_tab_url)
for cls in _ui_navbar_tabs
if cls.is_visible()
],
}
full_env.update(common)
full_env.update(env)
# Render it.
self.response.set_status(status)
self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
self.response.write(template.render(path, full_env))
def get(self):
self.response.write(template.render(
'adapter/proxy.html', params={'base_path': base_path}))
def render(name, params=None):
"""Shorthand to render a template."""
return template.render(name, params)
def reply(self, path, env=None, status=200):
"""Renders template |path| to the HTTP response using given environment.
Optional keys from |env| that base.html uses:
css_file: URL to a file with page specific styles, relative to site root.
js_file: URL to a file with page specific Javascript code, relative to
site root. File should define global object named same as a filename,
i.e. '/auth/static/js/api.js' should define global object 'api' that
incapsulates functionality implemented in the module.
navbar_tab_id: id of a navbar tab to highlight.
page_title: title of an HTML page.
Args:
path: path to a template, relative to templates/.
env: additional environment dict to use when rendering the template.
status: HTTP status code to return.
"""
env = (env or {}).copy()
env.setdefault('css_file', None)
env.setdefault('js_file', None)
env.setdefault('navbar_tab_id', None)
env.setdefault('page_title', 'Untitled')
# This goes to both Jinja2 env and Javascript config object.
user = self.get_current_user()
common = {
'account_picture': user.picture() if user else None,
'auth_service_config_locked': False, # overridden in auth_service
'is_admin': api.is_admin(),
'login_url': self.create_login_url(self.request.url),
'logout_url': self.create_logout_url('/'),
'using_gae_auth': self.auth_method == handler.gae_cookie_authentication,
'xsrf_token': self.generate_xsrf_token(),
}
if _ui_data_callback:
common.update(_ui_data_callback())
# Name of Javascript module with page code.
js_module_name = None
if env['js_file']:
assert env['js_file'].endswith('.js')
js_module_name = os.path.basename(env['js_file'])[:-3]
# This will be accessible from Javascript as global 'config' variable.
js_config = {
'identity': api.get_current_identity().to_bytes(),
}
js_config.update(common)
# Jinja2 environment to use to render a template.
full_env = {
'app_name': _ui_app_name,
'app_version': utils.get_app_version(),
'config': json.dumps(js_config),
'csp_nonce': self.csp_nonce,
'identity': api.get_current_identity(),
'js_module_name': js_module_name,
'navbar': [
(cls.navbar_tab_id, cls.navbar_tab_title, cls.navbar_tab_url)
for cls in _ui_navbar_tabs
if cls.is_visible()
],
}
full_env.update(common)
full_env.update(env)
# Render it.
self.response.set_status(status)
self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
self.response.write(template.render(path, full_env))
def get(self, request_id):
data = logscraper._log_request_id(request_id)
if not data:
self.abort(404, detail='Request id was not found.')
self.response.write(
template.render('ereporter2/request.html', {'request': data}))
def notify_gitiles_rejection(config_set, location, validation_result):
"""Notifies interested parties about an error in a config set revision.
Sends a notification per location only once.
Args:
location (gitiles.Location): an absolute gitiles location that could not be
imported.
validation_result (components.config.validation_context.Result).
"""
assert RE_GIT_HASH.match(location.treeish), location
if Notification.get_by_id(str(location)):
logging.debug('Notification was already sent.')
return
log = location.get_log(limit=1)
if not log or not log.commits:
logging.error('could not load commit %s', location)
return
commit = log.commits[0]
app_id = app_identity.get_application_id()
rev = location.treeish[:7]
try:
template_params = {
'author': commit.author.name or commit.author.email,
'messages': [
{
'severity': logging.getLevelName(msg.severity),
'text': msg.text
}
for msg in validation_result.messages
],
'rev_link': location,
'rev_hash': rev,
'rev_repo': location.project,
'cur_rev_hash': None,
'cur_rev_link': None,
}
cs = storage.ConfigSet.get_by_id(config_set)
if cs:
template_params.update(
cur_rev_hash=cs.latest_revision[:7],
cur_rev_link=location._replace(treeish=cs.latest_revision),
)
msg = mail.EmailMessage(
sender=(
'%s.appspot.com <*****@*****.**>' %
(app_id, app_id)),
subject='Config revision %s is rejected' % rev,
to=get_recipients(commit),
html=template.render(
'templates/validation_notification.html', template_params))
cc = get_cc_recipients()
if cc:
msg.cc = cc
logging.info('Emailing %s', ', '.join(msg.to))
_send(msg)
except mail_errors.Error as ex:
raise FailedToNotify(ex.message), None, sys.exc_info()[2]
Notification(id=str(location)).put()
def render(name, params=None): """Shorthand to render a template.""" return template.render(name, params)
