def silence(labels, duration=None, **kwargs):
'''
Post a silence message to Alert Manager
Duration should be sent in a format like 1m 2h 1d etc
'''
if duration:
start = timezone.now()
if duration.endswith('m'):
end = start + datetime.timedelta(minutes=int(duration[:-1]))
elif duration.endswith('h'):
end = start + datetime.timedelta(hours=int(duration[:-1]))
elif duration.endswith('d'):
end = start + datetime.timedelta(days=int(duration[:-1]))
else:
raise ValidationError('Unknown time modifier')
kwargs['endsAt'] = end.isoformat()
kwargs.pop('startsAt', False)
else:
local_timezone = pytz.timezone(util.setting("timezone", "UTC"))
for key in ['startsAt', 'endsAt']:
kwargs[key] = local_timezone.localize(parser.parse(
kwargs[key])).isoformat()
kwargs['matchers'] = [{
'name': name,
'value': value,
'isRegex': True if value.endswith("*") else False
} for name, value in labels.items()]
logger.debug("Sending silence for %s", kwargs)
url = urljoin(util.setting("alertmanager:url"), "/api/v1/silences")
response = util.post(url, json=kwargs)
response.raise_for_status()
return response
def settings_in_view(request):
return {
"EXTERNAL_LINKS":
util.setting("links", {}),
"TIMEZONE":
util.setting("timezone", "UTC"),
"VERSION":
__version__,
"DEFAULT_EXPORTERS":
models.DefaultExporter.objects.order_by("job", "-port"),
}
def check_rules(rules):
'''
Use promtool to check to see if a rule is valid or not
The command name changed slightly from 1.x -> 2.x but this uses promtool
to verify if the rules are correct or not. This can be bypassed by setting
a dummy command such as /usr/bin/true that always returns true
'''
with tempfile.NamedTemporaryFile(mode='w+b') as fp:
logger.debug('Rendering to %s', fp.name)
# Normally we wouldn't bother saving a copy to a variable here and would
# leave it in the fp.write() call, but saving a copy in the variable
# means we can see the rendered output in a Sentry stacktrace
rendered = render_rules(rules)
fp.write(rendered)
fp.flush()
# This command changed to be without a space in 2.x
cmd = [util.setting("prometheus:promtool"), "check", "rules", fp.name]
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
raise ValidationError(
rendered.decode('utf8') + e.output.decode('utf8'))
def reload_prometheus():
from promgen import signals
target = urljoin(util.setting("prometheus:url"), "/-/reload")
response = util.post(target)
response.raise_for_status()
signals.post_reload.send(response)
def delete(self, request, silence_id):
url = urljoin(util.setting("alertmanager:url"),
"/api/v1/silence/%s" % silence_id)
response = util.delete(url)
return HttpResponse(response.text,
status=response.status_code,
content_type="application/json")
def promtool(**kwargs):
try:
path = pathlib.Path(util.setting("prometheus:promtool"))
except TypeError:
yield checks.Warning("Missing setting for " + key)
else:
if not os.access(path, os.X_OK):
yield checks.Warning("Unable to execute file %s" % path)
def write_urls(path=None, reload=True, chmod=0o644):
if path is None:
path = util.setting("prometheus:blackbox")
with atomic_write(path, overwrite=True) as fp:
# Set mode on our temporary file before we write and move it
os.chmod(fp.name, chmod)
fp.write(prometheus.render_urls())
if reload:
reload_prometheus()
def get(self, request):
try:
url = urljoin(util.setting("alertmanager:url"), "/api/v1/silences")
response = util.get(url, params={"silenced": False})
except requests.exceptions.ConnectionError:
logger.error("Error connecting to %s", url)
return JsonResponse({})
else:
return HttpResponse(response.content,
content_type="application/json")
def directories(**kwargs):
for key in [
"prometheus:rules", "prometheus:blackbox", "prometheus:targets"
]:
try:
path = pathlib.Path(util.setting(key)).parent
except TypeError:
yield checks.Warning("Missing setting for " + key)
else:
if not os.access(path, os.W_OK):
yield checks.Warning("Unable to write to %s" % path)
def config(self, key, default=KeyError):
"""
Plugin specific configuration
This wraps our PROMGEN settings so that a plugin author does not need to
be concerned with how the configuration files are handled but only about
the specific key.
"""
try:
return util.setting(key, default=default, domain=self.__module__)
except KeyError as e:
raise KeyError(f"Missing key for domain: {self.__module__} {key}") from e
def promtool(**kwargs):
key = "prometheus:promtool"
try:
path = pathlib.Path(util.setting(key))
except TypeError:
yield checks.Warning(
"Missing setting for %s in %s " % (key, settings.PROMGEN_CONFIG_FILE),
id="promgen.W001",
)
else:
if not os.access(path, os.X_OK):
yield checks.Warning("Unable to execute file %s" % path, id="promgen.W003")
def directories(**kwargs):
for key in ["prometheus:rules", "prometheus:blackbox", "prometheus:targets"]:
try:
path = pathlib.Path(util.setting(key)).parent
except TypeError:
yield checks.Warning(
"Missing setting for %s in %s " % (key, settings.PROMGEN_CONFIG_FILE),
id="promgen.W001",
)
else:
if not os.access(path, os.W_OK):
yield checks.Warning("Unable to write to %s" % path, id="promgen.W002")
def config(self, key):
'''
Plugin specific configuration
This wraps our PROMGEN settings so that a plugin author does not need to
be concerned with how the configuration files are handled but only about
the specific key.
'''
try:
return util.setting(key, domain=self.__module__)
except KeyError:
logger.error(
'Undefined setting. Please check for %s under %s in settings.yml',
key, self.__module__)
def process_alert(alert_pk):
"""
Process alert for routing and notifications
We load our Alert from the database and expand it to determine which labels are routable
Next we loop through all senders configured and de-duplicate sender:target pairs before
queing the notification to actually be sent
"""
alert = models.Alert.objects.get(pk=alert_pk)
routable, data = alert.expand()
# For any blacklisted label patterns, we delete them from the queue
# and consider it done (do not send any notification)
blacklist = util.setting("alertmanager:blacklist", {})
for key in blacklist:
logger.debug("Checking key %s", key)
if key in data["commonLabels"]:
if data["commonLabels"][key] in blacklist[key]:
logger.debug("Blacklisted label %s", blacklist[key])
alert.delete()
return
# After processing our blacklist, it should be safe to queue our
# alert to also index the labels
index_alert.delay(alert.pk)
# Now that we have our routable items, we want to check which senders are
# configured and expand those as needed
senders = collections.defaultdict(set)
for label, obj in routable.items():
logger.debug("Processing %s %s", label, obj)
for sender in models.Sender.objects.filter(obj=obj, enabled=True):
if sender.filtered(data):
logger.debug("Filtered out sender %s", sender)
continue
if hasattr(sender.driver, "splay"):
for splay in sender.driver.splay(sender.value):
senders[splay.sender].add(splay.value)
else:
senders[sender.sender].add(sender.value)
for driver in senders:
for target in senders[driver]:
send_alert.delay(driver, target, data, alert.pk)
def strftime(timestamp, fmt):
tz = util.setting("timezone", "UTC")
if isinstance(timestamp, int) or isinstance(timestamp, float):
return timezone(tz).localize(datetime.fromtimestamp(timestamp)).strftime(fmt)
return timestamp
def clear_tombstones():
target = urljoin(util.setting("prometheus:url"),
"/api/v1/admin/tsdb/clean_tombstones")
response = util.post(target)
response.raise_for_status()
