def heartbeat(self, origin, tags=None, timeout=None, customer=None):
data = {
'origin': origin,
'tags': tags or list(),
'timeout': timeout,
'createTime': datetime.utcnow(),
'customer': customer
}
r = self.http.post('/heartbeat', data)
return Heartbeat.parse(r['heartbeat'])
def heartbeat(self, origin, tags=None, timeout=None, customer=None):
data = {
'origin': origin,
'tags': tags or list(),
'timeout': timeout,
'createTime': datetime.utcnow(),
'customer': customer
}
r = self.http.post('/heartbeat', data)
return Heartbeat.parse(r['heartbeat'])
def get_heartbeats(self, query=None):
r = self.http.get('/heartbeats', query)
return [Heartbeat.parse(hb) for hb in r['heartbeats']]
def get_heartbeat(self, id):
return Heartbeat.parse(
self.http.get('/heartbeat/%s' % id)['heartbeat'])
def get_heartbeats(self, query=None):
r = self.http.get('/heartbeats', query)
return [Heartbeat.parse(hb) for hb in r['heartbeats']]
def get_heartbeat(self, id):
return Heartbeat.parse(self.http.get('/heartbeat/%s' % id)['heartbeat'])
def cli(obj, alert, severity, timeout, purge):
"""List heartbeats."""
client = obj['client']
try:
default_normal_severity = obj['alarm_model']['defaults'][
'normal_severity']
except KeyError:
default_normal_severity = 'normal'
if obj['output'] == 'json':
r = client.http.get('/heartbeats')
heartbeats = [Heartbeat.parse(hb) for hb in r['heartbeats']]
click.echo(
json.dumps(r['heartbeats'],
sort_keys=True,
indent=4,
ensure_ascii=False))
else:
timezone = obj['timezone']
headers = {
'id': 'ID',
'origin': 'ORIGIN',
'customer': 'CUSTOMER',
'tags': 'TAGS',
'createTime': 'CREATED',
'receiveTime': 'RECEIVED',
'latency': 'LATENCY',
'timeout': 'TIMEOUT',
'since': 'SINCE',
'status': 'STATUS'
}
heartbeats = client.get_heartbeats()
click.echo(
tabulate([h.tabular(timezone) for h in heartbeats],
headers=headers,
tablefmt=obj['output']))
not_ok = [hb for hb in heartbeats if hb.status != 'ok']
if purge:
with click.progressbar(not_ok,
label='Purging {} heartbeats'.format(
len(not_ok))) as bar:
for b in bar:
client.delete_heartbeat(b.id)
if alert:
with click.progressbar(heartbeats,
label='Alerting {} heartbeats'.format(
len(heartbeats))) as bar:
for b in bar:
params = dict(
filter(lambda a: len(a) == 2,
map(lambda a: a.split(':'), b.tags)))
environment = params.get('environment', 'Production')
group = params.get('group', 'System')
tags = list(
filter(
lambda a: not a.startswith('environment:') and not a.
startswith('group:'), b.tags))
if b.status == 'expired': # aka. "stale"
client.send_alert(
resource=b.origin,
event='HeartbeatFail',
correlate=[
'HeartbeatFail', 'HeartbeatSlow', 'HeartbeatOK'
],
group=group,
environment=environment,
service=['Alerta'],
severity=severity,
value='{}'.format(b.since),
text='Heartbeat not received in {} seconds'.format(
b.timeout),
tags=tags,
type='heartbeatAlert',
timeout=timeout,
customer=b.customer)
elif b.status == 'slow':
client.send_alert(
resource=b.origin,
event='HeartbeatSlow',
correlate=[
'HeartbeatFail', 'HeartbeatSlow', 'HeartbeatOK'
],
group=group,
environment=environment,
service=['Alerta'],
severity=severity,
value='{}ms'.format(b.latency),
text='Heartbeat took more than {}ms to be processed'.
format(MAX_LATENCY),
tags=tags,
type='heartbeatAlert',
timeout=timeout,
customer=b.customer)
else:
client.send_alert(resource=b.origin,
event='HeartbeatOK',
correlate=[
'HeartbeatFail', 'HeartbeatSlow',
'HeartbeatOK'
],
group=group,
environment=environment,
service=['Alerta'],
severity=default_normal_severity,
value='',
text='Heartbeat OK',
tags=tags,
type='heartbeatAlert',
customer=b.customer)
def cli(obj, alert, severity, timeout, purge):
"""List heartbeats."""
client = obj['client']
try:
default_normal_severity = obj['alarm_model']['defaults']['normal_severity']
except KeyError:
default_normal_severity = 'normal'
if severity in ['normal', 'ok', 'cleared']:
raise click.UsageError('Must be a non-normal severity. "{}" is one of {}'.format(
severity, ', '.join(['normal', 'ok', 'cleared']))
)
if severity not in obj['alarm_model']['severity'].keys():
raise click.UsageError('Must be a valid severity. "{}" is not one of {}'.format(
severity, ', '.join(obj['alarm_model']['severity'].keys()))
)
if obj['output'] == 'json':
r = client.http.get('/heartbeats')
heartbeats = [Heartbeat.parse(hb) for hb in r['heartbeats']]
click.echo(json.dumps(r['heartbeats'], sort_keys=True, indent=4, ensure_ascii=False))
else:
timezone = obj['timezone']
headers = {
'id': 'ID', 'origin': 'ORIGIN', 'customer': 'CUSTOMER', 'tags': 'TAGS', 'attributes': 'ATTRIBUTES',
'createTime': 'CREATED', 'receiveTime': 'RECEIVED', 'since': 'SINCE', 'timeout': 'TIMEOUT',
'latency': 'LATENCY', 'maxLatency': 'MAX LATENCY', 'status': 'STATUS'
}
heartbeats = client.get_heartbeats()
click.echo(tabulate([h.tabular(timezone) for h in heartbeats], headers=headers, tablefmt=obj['output']))
not_ok = [hb for hb in heartbeats if hb.status != 'ok']
if purge:
with click.progressbar(not_ok, label=f'Purging {len(not_ok)} heartbeats') as bar:
for b in bar:
client.delete_heartbeat(b.id)
if alert:
with click.progressbar(heartbeats, label=f'Alerting {len(heartbeats)} heartbeats') as bar:
for b in bar:
want_environment = b.attributes.pop('environment', 'Production')
want_severity = b.attributes.pop('severity', severity)
want_service = b.attributes.pop('service', ['Alerta'])
want_group = b.attributes.pop('group', 'System')
if b.status == 'expired': # aka. "stale"
client.send_alert(
resource=b.origin,
event='HeartbeatFail',
environment=want_environment,
severity=want_severity,
correlate=['HeartbeatFail', 'HeartbeatSlow', 'HeartbeatOK'],
service=want_service,
group=want_group,
value=f'{b.since}',
text=f'Heartbeat not received in {b.timeout} seconds',
tags=b.tags,
attributes=b.attributes,
origin=origin(),
type='heartbeatAlert',
timeout=timeout,
customer=b.customer
)
elif b.status == 'slow':
client.send_alert(
resource=b.origin,
event='HeartbeatSlow',
environment=want_environment,
severity=want_severity,
correlate=['HeartbeatFail', 'HeartbeatSlow', 'HeartbeatOK'],
service=want_service,
group=want_group,
value=f'{b.latency}ms',
text=f'Heartbeat took more than {b.max_latency}ms to be processed',
tags=b.tags,
attributes=b.attributes,
origin=origin(),
type='heartbeatAlert',
timeout=timeout,
customer=b.customer
)
else:
client.send_alert(
resource=b.origin,
event='HeartbeatOK',
environment=want_environment,
severity=default_normal_severity,
correlate=['HeartbeatFail', 'HeartbeatSlow', 'HeartbeatOK'],
service=want_service,
group=want_group,
value='',
text='Heartbeat OK',
tags=b.tags,
attributes=b.attributes,
origin=origin(),
type='heartbeatAlert',
timeout=timeout,
customer=b.customer
)
def cli(obj, alert, severity, timeout, purge):
"""List heartbeats."""
client = obj['client']
try:
default_normal_severity = obj['alarm_model']['defaults']['normal_severity']
except KeyError:
default_normal_severity = 'normal'
if obj['output'] == 'json':
r = client.http.get('/heartbeats')
heartbeats = [Heartbeat.parse(hb) for hb in r['heartbeats']]
click.echo(json.dumps(r['heartbeats'], sort_keys=True, indent=4, ensure_ascii=False))
else:
timezone = obj['timezone']
headers = {
'id': 'ID', 'origin': 'ORIGIN', 'customer': 'CUSTOMER', 'tags': 'TAGS', 'createTime': 'CREATED',
'receiveTime': 'RECEIVED', 'latency': 'LATENCY', 'timeout': 'TIMEOUT', 'since': 'SINCE', 'status': 'STATUS'
}
heartbeats = client.get_heartbeats()
click.echo(tabulate([h.tabular(timezone) for h in heartbeats], headers=headers, tablefmt=obj['output']))
not_ok = [hb for hb in heartbeats if hb.status != 'ok']
if purge:
with click.progressbar(not_ok, label='Purging {} heartbeats'.format(len(not_ok))) as bar:
for b in bar:
client.delete_heartbeat(b.id)
if alert:
with click.progressbar(heartbeats, label='Alerting {} heartbeats'.format(len(heartbeats))) as bar:
for b in bar:
params = dict(filter(lambda a: len(a) == 2, map(lambda a: a.split(':'), b.tags)))
environment = params.get('environment', 'Production')
group = params.get('group', 'System')
tags = list(filter(lambda a: not a.startswith('environment:') and
not a.startswith('group:'), b.tags))
if b.status == 'expired': # aka. "stale"
client.send_alert(
resource=b.origin,
event='HeartbeatFail',
correlate=['HeartbeatFail', 'HeartbeatSlow', 'HeartbeatOK'],
group=group,
environment=environment,
service=['Alerta'],
severity=severity,
value='{}'.format(b.since),
text='Heartbeat not received in {} seconds'.format(b.timeout),
tags=tags,
type='heartbeatAlert',
timeout=timeout,
customer=b.customer
)
elif b.status == 'slow':
client.send_alert(
resource=b.origin,
event='HeartbeatSlow',
correlate=['HeartbeatFail', 'HeartbeatSlow', 'HeartbeatOK'],
group=group,
environment=environment,
service=['Alerta'],
severity=severity,
value='{}ms'.format(b.latency),
text='Heartbeat took more than {}ms to be processed'.format(MAX_LATENCY),
tags=tags,
type='heartbeatAlert',
timeout=timeout,
customer=b.customer
)
else:
client.send_alert(
resource=b.origin,
event='HeartbeatOK',
correlate=['HeartbeatFail', 'HeartbeatSlow', 'HeartbeatOK'],
group=group,
environment=environment,
service=['Alerta'],
severity=default_normal_severity,
value='',
text='Heartbeat OK',
tags=tags,
type='heartbeatAlert',
customer=b.customer
)
