def amtool_alert_query(self, mess, inhibited=False, silenced=False,
active=False, unprocessed=False, receiver="", matchers=[]):
"""
Queries alert in amtool command style
--inhibited Show inhibited alerts
--silenced Show silenced alerts
--active Show active alerts
--unprocessed Show unprocessed alerts
--receiver=RECEIVER Show alerts matching receiver (Supports regex syntax)
Args:
[<matcher-groups>] Query filter
"""
if not (inhibited or silenced or active or unprocessed):
inhibited = True
silenced = True
active = True
unprocessed = True
self.log.info("matchers {0}".format(matchers))
helper = AmtoolHelper(
alertmanager_address=self.config['server_address'])
filter = helper.get_filters_by_terms(matchers)
result = helper.get_alerts(
active=active,
silenced=silenced,
inhibited=inhibited,
unprocessed=unprocessed,
filter=filter,
receiver=receiver
)
return result
def amtool_status(self, mess, args):
"""Returns alert manager status"""
self.log.info("Current config {0}".format(self.config))
self.log.info(
"Alertmanager @ {0}".format(self.config['server_address']))
helper = AmtoolHelper(
alertmanager_address=self.config['server_address'])
result = helper.get_status()
return result
def amtool_silence_expire(self, mess, silence_id):
"""
amtool silence expire silence-id
expire an alertmanager silence
Args:
silence-id Id of silences to expire
"""
helper = AmtoolHelper(
alertmanager_address=self.config['server_address'])
result = helper.delete_silence(silence_id)
return "Silence deleted"
def amtool_alert_describe(self, mess, fingerprint):
"""Returns specific silence details"""
helper = AmtoolHelper(
alertmanager_address=self.config['server_address'])
result = helper.get_alert(fingerprint)
self.send_card(title=result["annotations"]["title"],
body=result["annotations"]["description"],
# thumbnail='https://raw.githubusercontent.com/errbotio/errbot/master/docs/_static/errbot.png',
# image='https://www.google.com/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png',
link=result["generatorURL"],
fields=result["labels"].items(),
color='blue',
in_reply_to=mess)
def test_post_silence_suppress(self):
amtoolhelper = AmtoolHelper(alertmanager_address=ALERTMANAGER_HOST)
start_period = datetime.now(pytz.timezone('Europe/Kiev'))
end_period = start_period + timedelta(minutes=1)
alert = amtoolhelper.get_alert('ee5d73c3f0a498f1')
matchers = amtoolhelper.get_matchers_by_alert(alert, ["name", "role"])
silence = amtoolhelper.post_silence(matchers=matchers,
starts_at=start_period.isoformat(),
ends_at=end_period.isoformat(),
created_by="Someone",
comment="test silence")
self.assertIsNotNone(silence)
def test_post_silence_selective(self):
amtoolhelper = AmtoolHelper(alertmanager_address=ALERTMANAGER_HOST)
start_period = datetime.now(pytz.timezone('Europe/Kiev'))
end_period = start_period + timedelta(minutes=1)
alert = amtoolhelper.get_alert('af2442fa7f7ee655')
matchers = amtoolhelper.get_matchers_by_terms(
terms=["~StrategyDown2", "instance=i-049a6b9bbbb6fb76b"])
silence = amtoolhelper.post_silence(matchers=matchers,
starts_at=start_period.isoformat(),
ends_at=end_period.isoformat(),
created_by="Someone",
comment="test silence")
self.assertIsNotNone(silence)
def amtool_silence_query(self, mess, expired=None, within=None, matchers=[]):
"""
Amtool has a simplified prometheus query syntax The non-option section of arguments constructs a list of "Matcher Groups"
that will be used to filter your query. The following examples will attempt to show this behaviour in action:
amtool silence query alertname=foo node=bar
This query will match all silences with the alertname=foo and node=bar label
value pairs set.
amtool silence query foo node=bar
If alertname is omitted and the first argument does not contain a '=' or a
'=~' then it will be assumed to be the value of the alertname pair.
amtool silence query 'alertname=~foo.*'
As well as direct equality, regex matching is also supported. The '=~' syntax
(similar to prometheus) is used to represent a regex match. Regex matching
can be used in combination with a direct match.
In addition to filtering by silence labels, one can also query for silences that are due to expire soon
with the "--within" parameter. In the event that you want to preemptively act upon expiring silences by
either fixing them or extending them. For example:
amtool silence query --within 8h
returns all the silences due to expire within the next 8 hours. This syntax can also be combined with the label based
filtering above for more flexibility.
The "--expired" parameter returns only expired silences. Used in combination with "--within=TIME", amtool returns
the silences that expired within the preceding duration.
amtool silence query --within 2h --expired
returns all silences that expired within the preceeding 2 hours.
--expired Show expired silences instead of active
--within=WITHIN Show silences that will expire or have expired within a duration
"""
helper = AmtoolHelper(
alertmanager_address=self.config['server_address'])
filters = helper.get_filters_by_terms(matchers)
self.log.info("Expired {0} within {1} filtered {2}".format(expired, within, filters))
result = helper.get_silences(filter=filters, expired=expired, within=within)
return {"silences": result}
def test_post_silence_add(self):
amtoolhelper = AmtoolHelper(alertmanager_address=ALERTMANAGER_HOST)
start_period = datetime.now().utcnow()
cal = pdt.Calendar()
diff = cal.parseDT("1h", sourceTime=datetime.min)[0] - datetime.min
end_period = start_period + diff
utc = pytz.UTC
start_period = utc.localize(start_period)
end_period = utc.localize(end_period)
parsed_matchers = amtoolhelper.get_matchers_by_terms(
["instance=i-049a6b9bbbb6fb76b"])
silence = amtoolhelper.post_silence(matchers=parsed_matchers,
starts_at=start_period.isoformat(),
ends_at=end_period.isoformat(),
created_by="errbot",
comment="test comment")
self.assertIsNotNone(silence)
def amtool_suppress(self, mess, author="errbot", comment="", weeks=0, days=0, hours=0, minutes=0, criteria=[]):
"""Puts exact suppress match on alert"""
helper = AmtoolHelper(
alertmanager_address=self.config['server_address'])
start_period = datetime.now(pytz.timezone(self.config['time_zone']))
if weeks+days+hours+minutes == 0:
hours = 1
end_period = start_period + timedelta(minutes=minutes, hours=hours,
days=days, weeks=weeks)
self.log.info("Suppressing {0}->{1}".format(start_period, end_period))
fingerprint = criteria.pop(0)
self.log.info("Getting alert by fingerprint {0}".format(fingerprint))
alert = helper.get_alert(fingerprint)
matchers = helper.get_matchers_by_alert(alert, include_terms=criteria)
self.log.info("Matchers {0}".format(matchers))
result = helper.post_silence(
matchers=matchers,
starts_at=start_period.isoformat(),
ends_at=end_period.isoformat(),
created_by=author,
comment=comment
)
self.send_card(title="Alert suppressed until {0}".format(end_period),
body="Alert created by {0} with description '{1}'. To cancel !amtool silence expire {2}".format(author, comment, result.silence_id),
# thumbnail='https://raw.githubusercontent.com/errbotio/errbot/master/docs/_static/errbot.png',
# image='https://www.google.com/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png',
# link=result["generatorURL"],
fields=helper.convert_matchers_to_tuples(matchers),
color='blue',
in_reply_to=mess)
def amtool_brief(self, mess, args):
"""Returns brief on alerts"""
helper = AmtoolHelper(
alertmanager_address=self.config['server_address'])
result = helper.get_alerts()
return result
def test_get_status(self):
amtoolhelper = AmtoolHelper(alertmanager_address=ALERTMANAGER_HOST)
amtoolhelper.get_status()
def amtool_silence_describe(self, mess, silence_id):
"""Returns specific silence details"""
helper = AmtoolHelper(
alertmanager_address=self.config['server_address'])
result = helper.get_silence(silence_id)
return result
def amtool_receivers(self, mess, args):
"""Returns current receivers list"""
helper = AmtoolHelper(
alertmanager_address=self.config['server_address'])
result = helper.get_receivers()
return result
def test_get_alerts(self):
amtoolhelper = AmtoolHelper(alertmanager_address=ALERTMANAGER_HOST)
alerts = amtoolhelper.get_alerts()
self.assertIsNotNone(alerts)
def amtool_silences(self, mess, args):
"""Returns current silences list"""
helper = AmtoolHelper(
alertmanager_address=self.config['server_address'])
result = helper.get_silences()
return {"silences": result}
def test_get_alert(self):
amtoolhelper = AmtoolHelper(alertmanager_address=ALERTMANAGER_HOST)
alert = amtoolhelper.get_alert('af2442fa7f7ee655')
self.assertIsNotNone(alert)
def test_get_alerts_with_filter(self):
amtoolhelper = AmtoolHelper(alertmanager_address=ALERTMANAGER_HOST)
filter = amtoolhelper.get_filters_by_terms(["env=live"])
alerts = amtoolhelper.get_alerts(filter=filter)
self.assertIsNotNone(alerts)
def test_get_silences(self):
amtoolhelper = AmtoolHelper(alertmanager_address=ALERTMANAGER_HOST)
silences = amtoolhelper.get_silences(filter=[])
self.assertIsNotNone(silences)
def amtool_silence_add(self, mess, author, duration, start, end, comment,
matchers):
"""
usage: !amtool silence add [<flags>] [<matcher-groups>...]
Add a new alertmanager silence
Amtool uses a simplified Prometheus syntax to represent silences. The
non-option section of arguments constructs a list of "Matcher Groups"
that will be used to create a number of silences. The following examples
will attempt to show this behaviour in action:
!amtool silence add alertname=foo node=bar
This statement will add a silence that matches alerts with the
alertname=foo and node=bar label value pairs set.
!amtool silence add foo node=bar
If alertname is omitted and the first argument does not contain a '=' or a
'=~' then it will be assumed to be the value of the alertname pair.
!amtool silence add 'alertname=~foo.*'
As well as direct equality, regex matching is also supported. The '=~' syntax
(similar to Prometheus) is used to represent a regex match. Regex matching
can be used in combination with a direct match.
--author="slavko" Username for CreatedBy field
--duration="1h" Duration of silence
--start=START Set when the silence should start. RFC3339 format 2006-01-02T15:04:05-07:00
--end=END Set when the silence should end (overwrites duration). RFC3339 format 2006-01-02T15:04:05-07:00
--comment=COMMENT A comment to help describe the silence
"""
helper = AmtoolHelper(
alertmanager_address=self.config['server_address'])
if start is not None:
start_period = dateparser.parse(start)
else:
start_period = datetime.now().utcnow()
if end is not None:
end_period = dateparser.parse(end)
else:
cal = pdt.Calendar()
diff = cal.parseDT(duration, sourceTime=datetime.min)[
0] - datetime.min
end_period = start_period + diff
utc = pytz.UTC
start_period = utc.localize(start_period)
end_period = utc.localize(end_period)
parsed_matchers = helper.get_matchers_by_terms(matchers)
self.log.info("Suppressing {0}->{1}".format(start_period, end_period))
self.log.info("Matchers {0}".format(parsed_matchers))
result = helper.post_silence(
matchers=parsed_matchers,
starts_at=start_period.isoformat(),
ends_at=end_period.isoformat(),
created_by=author,
comment=comment
)
self.log.info("Added {0}".format(result))
self.send_card(title="Silence added until {0}".format(end_period),
body="Alert created by {0} with description '{1}'. To cancel !amtool silence expire {2}".format(author, comment, result.silence_id),
# thumbnail='https://raw.githubusercontent.com/errbotio/errbot/master/docs/_static/errbot.png',
# image='https://www.google.com/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png',
# link=result["generatorURL"],
fields=helper.convert_matchers_to_tuples(parsed_matchers),
color='blue',
in_reply_to=mess)
def test_post_silence_selective(self):
amtoolhelper = AmtoolHelper(alertmanager_address=ALERTMANAGER_HOST)
parsed_matchers = amtoolhelper.get_matchers_by_terms(["instance=222"])
tuples = amtoolhelper.convert_matchers_to_tuples(parsed_matchers)
def test_expired_within_silences(self):
amtoolhelper = AmtoolHelper(alertmanager_address=ALERTMANAGER_HOST)
silences = amtoolhelper.get_silences(filter=[],
expired=True,
within="1h")
self.assertIsNotNone(silences)
def test_get_silence(self):
amtoolhelper = AmtoolHelper(alertmanager_address=ALERTMANAGER_HOST)
silence = amtoolhelper.get_silence(
silence_id="bb788860-35d2-48e7-9062-f082c77d202d")
self.assertIsNotNone(silence)