def post(self, subdomain, method = None, alert_id = None):
form = AlertForm(MultiDict(self))
if not form.validate():
args = self.template_args
args['form'] = form
args['method'] = method
args['subdomain'] = subdomain
return self.render('alert.html', **args)
user = UserManager.get_current_user()
if method == 'create':
alert = Alert()
else:
alert = AlertManager.get_alert(alert_id)
alert.email = user.email
alert.subdomain = user.subdomain
alert.name = self.get_argument('name', '')
alert.description = self.get_argument('description', '')
alert.saved_search = int(self.get_argument('saved_search'))
alert.threshold_operator = self.get_argument('threshold_operator')
alert.threshold_count = int(self.get_argument('threshold_count'))
alert.threshold_time_secs = int(self.get_argument('threshold_time_secs'))
alert.sound = self.get_argument('sound')
alert.endpoint = self.get_argument('endpoint', '')
if method == 'create':
alert.active = True
alert.muted = False
alert.state = 'N'
alert.last_run = 0
alert.last_state_change = 0
alert.put()
self.redirect('/%s' % subdomain)
def get(self, subdomain, method = None, alert_id = None):
args = self.template_args
if method == 'create':
args['form'] = AlertForm()
else:
alert = AlertManager.get_alert(alert_id)
args['form'] = AlertForm(alert = alert)
args['method'] = method
args['subdomain'] = subdomain
self.render('alert.html', **args)
def get(self, subdomain, action = None, alert_id = None):
alert = AlertManager.get_alert(alert_id)
if action == 'mute':
alert.muted = True
alert.put()
elif action == 'unmute':
alert.muted = False
alert.put()
if action == 'enable':
alert.active = True
alert.put()
if action == 'disable':
alert.active = False
alert.put()
if action == 'getssurl':
saved_search = SavedSearchManager.get_saved_search(alert.saved_search)
self.write(Loggly.build_search_url(saved_search.context, alert.threshold_time_secs))
elif action == 'delete':
alert.delete()
def get(self, subdomain, action=None, alert_id=None):
alert = AlertManager.get_alert(alert_id)
if action == 'mute':
alert.muted = True
alert.put()
elif action == 'unmute':
alert.muted = False
alert.put()
if action == 'enable':
alert.active = True
alert.put()
if action == 'disable':
alert.active = False
alert.put()
if action == 'getssurl':
saved_search = SavedSearchManager.get_saved_search(
alert.saved_search)
self.write(
Loggly.build_search_url(saved_search.context,
alert.threshold_time_secs))
elif action == 'delete':
alert.delete()
def get(self, token, prefix):
if token != config.CRON_PASSWORD:
self.write('nope')
return
else:
# App Engine cron jobs don't always run exactly on the minute,
# so make sure all times are evenly divisible by 60
run_time = (int(math.floor(time.time())) // 60) * 60
alerts = AlertManager.get_all_alerts_systemwide(prefix=prefix)
for alert in alerts:
if alert.active == False:
continue
# TODO what if this times out?
if alert.last_run == 0 or alert.state == 'C' or (
run_time -
alert.last_run) >= alert.threshold_time_secs:
# this is a blanket try/catch so misconfigured endpoints, etc. don't impact other alerts.
try:
user = UserManager.get_user(alert.email)
oauth_client = lib.oauth.Client(user.subdomain)
token = oauth_client.generate_token(
user.access_token_key, user.access_token_secret)
# to create a dummy saved search, POST to
# http://davidlanstein.frontend-david1.office.loggly.net/api/savedsearches/create
# with this data:
# name=foo&context={"search_type":"search", "terms":"ivan tam", "from":"NOW-1DAY", "until":"NOW", "inputs":["logglyapp","logglyweb"], "order":"desc", "buckets": null, "highlighting":true, "rows":20, "start":0, "page":0, "command_string":null}
saved_searches = SavedSearchManager.get_all_saved_searches(
user)
found = False
for saved_search in saved_searches:
if saved_search.id == alert.saved_search:
found = True
break
if not found:
# search was deleted, perhaps?
logging.warn({
'module':
'controllers.cron',
'message':
'Alert with id \'%s\' is associated with saved search \'%s\', which no longer exists.'
% (unicode(alert.key()), saved_search.id)
})
continue
qs = Loggly.build_search_query_string(
saved_search.context, alert.threshold_time_secs)
try:
search_result = json.loads(oauth_client.make_request(token, 'http://%s.%s/api/facets/date?%s' % \
(user.subdomain, config.LOGGLY_DOMAIN, qs), 'GET'))
except Exception, e:
logging.error({
'module': 'controllers.cron',
'traceback': traceback.format_exc()
})
# input name in saved search doesn't exist anymore, etc.
continue
if alert.threshold_operator == 'gt':
fire_alert = search_result[
'numFound'] > alert.threshold_count
elif alert.threshold_operator == 'lt':
fire_alert = search_result[
'numFound'] < alert.threshold_count
else:
fire_alert = search_result[
'numFound'] == alert.threshold_count
if not fire_alert:
if alert.state == 'C':
alert.state = 'N'
alert.last_state_change = run_time
alert_json = {
'sound':
alert.sound,
'description':
tornado.escape.xhtml_escape(
alert.description),
'name':
tornado.escape.xhtml_escape(alert.name),
'state':
alert.state,
'key':
unicode(alert.key()),
'muted':
alert.muted,
'last_state_change':
alert.last_state_change
}
alert_channel = hashlib.md5(
'alertbirds' +
alert.subdomain).hexdigest()
pusher_client = pusher.Pusher(
app_id=config.PUSHER_APP_ID,
key=config.PUSHER_KEY,
secret=config.PUSHER_SECRET)
result = pusher_client[alert_channel].trigger(
'chirp', data=alert_json)
if alert.endpoint:
endpoint = EndpointManager.get_endpoint(
alert.endpoint, alert.email)
pagerduty = PagerDuty(endpoint.service_key)
pagerduty.resolve(unicode(alert.key()))
else:
if alert.state == 'N':
alert.state = 'C'
alert.last_state_change = run_time
logging.warn({
'module':
'controllers.cron',
'message':
'Alert with id \'%s\' is in a critical state.'
% unicode(alert.key())
})
alert_json = {
'sound':
alert.sound,
'description':
tornado.escape.xhtml_escape(alert.description),
'name':
tornado.escape.xhtml_escape(alert.name),
'state':
alert.state,
'key':
unicode(alert.key()),
'muted':
alert.muted,
'last_state_change':
alert.last_state_change
}
alert_channel = hashlib.md5(
'alertbirds' + alert.subdomain).hexdigest()
pusher_client = pusher.Pusher(
app_id=config.PUSHER_APP_ID,
key=config.PUSHER_KEY,
secret=config.PUSHER_SECRET)
result = pusher_client[alert_channel].trigger(
'chirp', data=alert_json)
if alert.endpoint:
endpoint = EndpointManager.get_endpoint(
alert.endpoint, alert.email)
pagerduty = PagerDuty(endpoint.service_key)
pagerduty.trigger(endpoint.alert_text,
unicode(alert.key()),
alert.description)
# if pagerduty is experiencing an outage, still re-run next minute
# that's why we set last_run at the bottom
alert.last_run = run_time
alert.put()
except Exception, e:
# endpoint misconfigured, who knows what else. don't impact other users.
logging.error({
'module': 'controllers.cron',
'traceback': traceback.format_exc()
})
def get(self, token, prefix):
if token != config.CRON_PASSWORD:
self.write('nope')
return
else:
# App Engine cron jobs don't always run exactly on the minute,
# so make sure all times are evenly divisible by 60
run_time = (int(math.floor(time.time())) // 60) * 60
alerts = AlertManager.get_all_alerts_systemwide(prefix=prefix)
for alert in alerts:
if alert.active == False:
continue
# TODO what if this times out?
if alert.last_run == 0 or alert.state == 'C' or (run_time - alert.last_run) >= alert.threshold_time_secs:
# this is a blanket try/catch so misconfigured endpoints, etc. don't impact other alerts.
try:
user = UserManager.get_user(alert.email)
oauth_client = lib.oauth.Client(user.subdomain)
token = oauth_client.generate_token(user.access_token_key, user.access_token_secret)
# to create a dummy saved search, POST to
# http://davidlanstein.frontend-david1.office.loggly.net/api/savedsearches/create
# with this data:
# name=foo&context={"search_type":"search", "terms":"ivan tam", "from":"NOW-1DAY", "until":"NOW", "inputs":["logglyapp","logglyweb"], "order":"desc", "buckets": null, "highlighting":true, "rows":20, "start":0, "page":0, "command_string":null}
saved_searches = SavedSearchManager.get_all_saved_searches(user)
found = False
for saved_search in saved_searches:
if saved_search.id == alert.saved_search:
found = True
break
if not found:
# search was deleted, perhaps?
logging.warn({'module': 'controllers.cron', 'message': 'Alert with id \'%s\' is associated with saved search \'%s\', which no longer exists.' % (unicode(alert.key()), saved_search.id)})
continue
qs = Loggly.build_search_query_string(saved_search.context, alert.threshold_time_secs)
try:
search_result = json.loads(oauth_client.make_request(token, 'http://%s.%s/api/facets/date?%s' % \
(user.subdomain, config.LOGGLY_DOMAIN, qs), 'GET'))
except Exception, e:
logging.error({'module': 'controllers.cron', 'traceback': traceback.format_exc()})
# input name in saved search doesn't exist anymore, etc.
continue
if alert.threshold_operator == 'gt':
fire_alert = search_result['numFound'] > alert.threshold_count
elif alert.threshold_operator == 'lt':
fire_alert = search_result['numFound'] < alert.threshold_count
else:
fire_alert = search_result['numFound'] == alert.threshold_count
if not fire_alert:
if alert.state == 'C':
alert.state = 'N'
alert.last_state_change = run_time
alert_json = {'sound': alert.sound , 'description': tornado.escape.xhtml_escape(alert.description), 'name': tornado.escape.xhtml_escape(alert.name), 'state': alert.state, 'key': unicode(alert.key()), 'muted': alert.muted, 'last_state_change': alert.last_state_change }
alert_channel = hashlib.md5('alertbirds' + alert.subdomain).hexdigest()
pusher_client = pusher.Pusher(app_id=config.PUSHER_APP_ID, key=config.PUSHER_KEY, secret=config.PUSHER_SECRET)
result = pusher_client[alert_channel].trigger('chirp', data=alert_json)
if alert.endpoint:
endpoint = EndpointManager.get_endpoint(alert.endpoint, alert.email)
pagerduty = PagerDuty(endpoint.service_key)
pagerduty.resolve(unicode(alert.key()))
else:
if alert.state == 'N':
alert.state = 'C'
alert.last_state_change = run_time
logging.warn({'module': 'controllers.cron', 'message': 'Alert with id \'%s\' is in a critical state.' % unicode(alert.key())})
alert_json = {'sound': alert.sound , 'description': tornado.escape.xhtml_escape(alert.description), 'name': tornado.escape.xhtml_escape(alert.name), 'state': alert.state, 'key': unicode(alert.key()), 'muted': alert.muted, 'last_state_change': alert.last_state_change }
alert_channel = hashlib.md5('alertbirds' + alert.subdomain).hexdigest()
pusher_client = pusher.Pusher(app_id=config.PUSHER_APP_ID, key=config.PUSHER_KEY, secret=config.PUSHER_SECRET)
result = pusher_client[alert_channel].trigger('chirp', data=alert_json)
if alert.endpoint:
endpoint = EndpointManager.get_endpoint(alert.endpoint, alert.email)
pagerduty = PagerDuty(endpoint.service_key)
pagerduty.trigger(endpoint.alert_text, unicode(alert.key()), alert.description)
# if pagerduty is experiencing an outage, still re-run next minute
# that's why we set last_run at the bottom
alert.last_run = run_time
alert.put()
except Exception, e:
# endpoint misconfigured, who knows what else. don't impact other users.
logging.error({'module': 'controllers.cron', 'traceback': traceback.format_exc()})
def get(self, subdomain=None):
'''
We're here for one of several reasons:
1. Initial visit, not logged in, going to '/' (show welcome page)
2. Initial visit, no User object stored yet in App Engine datastore (returning from google.com)
3. Return visit, everything normal
4. Return visit, no subdomain in URL (so going to '/')
5. User object stored, but access token not yet written (returning from loggly.com)
6. User object stored, but access token not yet written (App Engine threw exception when returning from Loggly, etc.)
7. User entered a subdomain, then hit the back button
'''
args = self.template_args
if not google_users.get_current_user():
# case 1
logging.info({'module': 'main', 'path': 'case 1'})
args['login_url'] = google_users.create_login_url(self.request.uri)
return self.render('splash.html', **args)
if not UserManager.current_user_exists():
# case 2
logging.info({'module': 'main', 'path': 'case 2'})
args['form'] = SubdomainForm()
return self.render('subdomain.html', **args)
user = UserManager.get_current_user()
if subdomain:
args['subdomain'] = subdomain
else:
args['subdomain'] = user.subdomain
if self.get_argument('oauth_verifier',
None) or not user.access_token_key:
# cases 5, 6, or 7
logging.info({'module': 'main', 'path': 'case 5, 6 or 7'})
oauth_client = lib.oauth.Client(user.subdomain)
if self.get_argument('oauth_verifier', None):
# case 5 (just in case)
user.oauth_verifier = self.get_argument('oauth_verifier')
user.put()
elif not user.oauth_verifier:
# case 7
request_token = oauth_client.generate_token(
user.request_token_key, user.request_token_secret)
url = oauth_client.get_authorize_url(request_token)
return self.redirect(url)
request_token = oauth_client.generate_token(
user.request_token_key, user.request_token_secret)
request_token.verifier = user.oauth_verifier
access_token = oauth_client.get_access_token(request_token)
# store the access token for all future requests
user.access_token_key = access_token.key
user.access_token_secret = access_token.secret
user.new_user = False
user.put()
search_result = json.loads(oauth_client.make_request(access_token, 'http://%s.%s/api/facets/date?q=*&from=NOW-1YEAR' % \
(user.subdomain, config.LOGGLY_DOMAIN), 'GET'))
logging.info(
json.dumps({
'module': 'main',
'user': user.email,
'event': 'new user',
'loggly_events_found': search_result['numFound']
}))
if search_result['numFound'] == 0:
return self.render('nodata.html', **args)
if not subdomain:
# case 4
logging.info({'module': 'main', 'path': 'case 4'})
# this must come after the others so we don't lose the OAuth parameters from the URL
return self.redirect('/%s' % user.subdomain)
# case 3 or 5
logging.info({'module': 'main', 'path': 'case 3 or 5'})
if self.request.uri[-1] == '/':
return self.redirect(self.request.uri[:-1])
args['alerts'] = AlertManager.get_all_alerts(user.subdomain)
self.render('main.html', **args)
def get(self, subdomain = None):
'''
We're here for one of several reasons:
1. Initial visit, not logged in, going to '/' (show welcome page)
2. Initial visit, no User object stored yet in App Engine datastore (returning from google.com)
3. Return visit, everything normal
4. Return visit, no subdomain in URL (so going to '/')
5. User object stored, but access token not yet written (returning from loggly.com)
6. User object stored, but access token not yet written (App Engine threw exception when returning from Loggly, etc.)
7. User entered a subdomain, then hit the back button
'''
args = self.template_args
if not google_users.get_current_user():
# case 1
logging.info({'module': 'main', 'path': 'case 1'})
args['login_url'] = google_users.create_login_url(self.request.uri)
return self.render('splash.html', **args)
if not UserManager.current_user_exists():
# case 2
logging.info({'module': 'main', 'path': 'case 2'})
args['form'] = SubdomainForm()
return self.render('subdomain.html', **args)
user = UserManager.get_current_user()
if subdomain:
args['subdomain'] = subdomain
else:
args['subdomain'] = user.subdomain
if self.get_argument('oauth_verifier', None) or not user.access_token_key:
# cases 5, 6, or 7
logging.info({'module': 'main', 'path': 'case 5, 6 or 7'})
oauth_client = lib.oauth.Client(user.subdomain)
if self.get_argument('oauth_verifier', None):
# case 5 (just in case)
user.oauth_verifier = self.get_argument('oauth_verifier')
user.put()
elif not user.oauth_verifier:
# case 7
request_token = oauth_client.generate_token(user.request_token_key, user.request_token_secret)
url = oauth_client.get_authorize_url(request_token)
return self.redirect(url)
request_token = oauth_client.generate_token(user.request_token_key, user.request_token_secret)
request_token.verifier = user.oauth_verifier
access_token = oauth_client.get_access_token(request_token)
# store the access token for all future requests
user.access_token_key = access_token.key
user.access_token_secret = access_token.secret
user.new_user = False
user.put()
search_result = json.loads(oauth_client.make_request(access_token, 'http://%s.%s/api/facets/date?q=*&from=NOW-1YEAR' % \
(user.subdomain, config.LOGGLY_DOMAIN), 'GET'))
logging.info(json.dumps({'module': 'main', 'user': user.email, 'event': 'new user', 'loggly_events_found': search_result['numFound']}))
if search_result['numFound'] == 0:
return self.render('nodata.html', **args)
if not subdomain:
# case 4
logging.info({'module': 'main', 'path': 'case 4'})
# this must come after the others so we don't lose the OAuth parameters from the URL
return self.redirect('/%s' % user.subdomain)
# case 3 or 5
logging.info({'module': 'main', 'path': 'case 3 or 5'})
if self.request.uri[-1] == '/':
return self.redirect(self.request.uri[:-1])
args['alerts'] = AlertManager.get_all_alerts(user.subdomain)
self.render('main.html', **args)
