def save_heartbeat(self, tenant, heartbeat):
now = datetime.datetime.utcnow()
dBase = self._client[tenant]
update = {
# '$setOnInsert': {"_id": heartbeat.id},
'$set': {
"origin": heartbeat.origin,
"tags": heartbeat.tags,
"type": heartbeat.event_type,
"createTime": heartbeat.create_time,
"timeout": heartbeat.timeout,
"receiveTime": now,
"customer": heartbeat.customer
}
}
LOG.debug('Save heartbeat to database: %s', update)
heartbeat_id = dBase.heartbeats.find_one({"origin": heartbeat.origin}, {})
if heartbeat_id:
response = dBase.heartbeats.find_one_and_update(
{"origin": heartbeat.origin},
update=update,
upsert=True,
return_document=ReturnDocument.AFTER
)
return HeartbeatDocument(
id=response['_id'],
origin=response['origin'],
tags=response['tags'],
event_type=response['type'],
create_time=response['createTime'],
timeout=response['timeout'],
receive_time=response['receiveTime'],
customer=response.get('customer', None)
)
else:
update = update['$set']
update["_id"] = heartbeat.id
response = dBase.heartbeats.insert_one(update)
return HeartbeatDocument(
id=response.inserted_id,
origin=update['origin'],
tags=update['tags'],
event_type=update['type'],
create_time=update['createTime'],
timeout=update['timeout'],
receive_time=update['receiveTime'],
customer=update.get('customer', None)
)
def save_heartbeat(self, heartbeat):
now = datetime.datetime.utcnow()
update = {
# '$setOnInsert': {"_id": heartbeat.id},
'$set': {
"origin": heartbeat.origin,
"tags": heartbeat.tags,
"type": heartbeat.event_type,
"createTime": heartbeat.create_time,
"timeout": heartbeat.timeout,
"receiveTime": now
}
}
LOG.debug('Save heartbeat to database: %s', update)
heartbeat_id = self.db.heartbeats.find_one(
{"origin": heartbeat.origin}, {})
if heartbeat_id:
no_obj_error = "No matching object found"
response = self.db.command("findAndModify",
'heartbeats',
allowable_errors=[no_obj_error],
query={"origin": heartbeat.origin},
update=update,
new=True,
upsert=True)["value"]
return HeartbeatDocument(id=response['_id'],
origin=response['origin'],
tags=response['tags'],
event_type=response['type'],
create_time=response['createTime'],
timeout=response['timeout'],
receive_time=response['receiveTime'])
else:
update = update['$set']
update["_id"] = heartbeat.id
response = self.db.heartbeats.insert(update)
return HeartbeatDocument(id=response,
origin=update['origin'],
tags=update['tags'],
event_type=update['type'],
create_time=update['createTime'],
timeout=update['timeout'],
receive_time=update['receiveTime'])
def get_heartbeat(self, id):
if len(id) == 8:
query = {
'$or': [{
'_id': {
'$regex': '^' + id
}
}, {
'lastReceiveId': {
'$regex': '^' + id
}
}]
}
else:
query = {'$or': [{'_id': id}, {'lastReceiveId': id}]}
response = self.db.heartbeats.find_one(query)
if not response:
return
return HeartbeatDocument(id=response['_id'],
tags=response['tags'],
origin=response['origin'],
event_type=response['type'],
create_time=response['createTime'],
timeout=response['timeout'],
receive_time=response['receiveTime'])
def get_heartbeats(self):
responses = self.db.heartbeats.find()
heartbeats = list()
for response in responses:
heartbeats.append(
HeartbeatDocument(id=response['_id'],
origin=response['origin'],
tags=response['tags'],
event_type=response['type'],
create_time=response['createTime'],
timeout=response['timeout'],
receive_time=response['receiveTime']))
return heartbeats
def get_heartbeats(self, tenant):
dBase = self._client[tenant]
responses = dBase.heartbeats.find()
heartbeats = list()
for response in responses:
heartbeats.append(
HeartbeatDocument(
id=response['_id'],
origin=response['origin'],
tags=response['tags'],
event_type=response['type'],
create_time=response['createTime'],
timeout=response['timeout'],
receive_time=response['receiveTime'],
customer=response.get('customer', None)
)
)
return heartbeats
def get_heartbeat(self, tenant, id):
dBase = self._client[tenant]
if len(id) == 8:
query = {'$or': [{'_id': {'$regex': '^' + id}}, {'lastReceiveId': {'$regex': '^' + id}}]}
else:
query = {'$or': [{'_id': id}, {'lastReceiveId': id}]}
response = dBase.heartbeats.find_one(query)
if not response:
return
return HeartbeatDocument(
id=response['_id'],
tags=response['tags'],
origin=response['origin'],
event_type=response['type'],
create_time=response['createTime'],
timeout=response['timeout'],
receive_time=response['receiveTime'],
customer=response.get('customer', None)
)
def heartbeats(self, args):
response = self._heartbeats()
heartbeats = response['heartbeats']
print('{:<28} {:<26} {:<19} {:>8} {:7} {}'.format(
'ORIGIN', 'TAGS', 'CREATED', 'LATENCY', 'TIMEOUT', 'SINCE'))
for heartbeat in heartbeats:
hb = HeartbeatDocument.parse_heartbeat(heartbeat)
latency = (hb.receive_time - hb.create_time).microseconds / 1000
since = datetime.utcnow() - hb.receive_time
since = since - timedelta(microseconds=since.microseconds)
latency_exceeded = latency > MAX_LATENCY
timeout_exceeded = since.seconds > hb.timeout
print('{:<28} {:<26} {} {}{:6}ms {:6}s {}{}'.format(
hb.origin, ' '.join(hb.tags),
hb.get_date('create_time', 'local',
args.timezone), '*' if latency_exceeded else ' ',
latency, hb.timeout, '*' if timeout_exceeded else ' ', since))
if args.alert:
if timeout_exceeded:
alert = Alert(
resource=hb.origin,
event='HeartbeatFail',
correlate=[
'HeartbeatFail', 'HeartbeatSlow', 'HeartbeatOK'
],
group='System',
environment='Production',
service=['Alerta'],
severity='major',
value='{}'.format(since),
text='Heartbeat not received in {} seconds'.format(
hb.timeout),
tags=hb.tags,
type='heartbeatAlert')
elif latency_exceeded:
alert = Alert(
resource=hb.origin,
event='HeartbeatSlow',
correlate=[
'HeartbeatFail', 'HeartbeatSlow', 'HeartbeatOK'
],
group='System',
environment='Production',
service=['Alerta'],
severity='major',
value='{}ms'.format(latency),
text='Heartbeat took more than {}ms to be processed'.
format(MAX_LATENCY),
tags=hb.tags,
type='heartbeatAlert')
else:
alert = Alert(resource=hb.origin,
event='HeartbeatOK',
correlate=[
'HeartbeatFail', 'HeartbeatSlow',
'HeartbeatOK'
],
group='System',
environment='Production',
service=['Alerta'],
severity='normal',
value='',
text='Heartbeat OK',
tags=hb.tags,
type='heartbeatAlert')
self.send(alert)
def heartbeats(self, args):
response = self._heartbeats()
heartbeats = response['heartbeats']
print('{:<28} {:<26} {:<19} {:>8} {:7} {}'.format('ORIGIN', 'TAGS', 'CREATED', 'LATENCY', 'TIMEOUT', 'SINCE'))
for heartbeat in heartbeats:
hb = HeartbeatDocument.parse_heartbeat(heartbeat)
latency = (hb.receive_time - hb.create_time).microseconds / 1000
since = datetime.utcnow() - hb.receive_time
since = since - timedelta(microseconds=since.microseconds)
latency_exceeded = latency > MAX_LATENCY
timeout_exceeded = since.seconds > hb.timeout
print('{:<28} {:<26} {} {}{:6}ms {:6}s {}{}'.format(
hb.origin,
' '.join(hb.tags),
hb.get_date('create_time', 'local', args.timezone),
'*' if latency_exceeded else ' ', latency,
hb.timeout,
'*' if timeout_exceeded else ' ', since
))
if args.alert:
if timeout_exceeded:
alert = Alert(
resource=hb.origin,
event='HeartbeatFail',
correlate=['HeartbeatFail', 'HeartbeatSlow', 'HeartbeatOK'],
group='System',
environment='Production',
service=['Alerta'],
severity='major',
value='{}'.format(since),
text='Heartbeat not received in {} seconds'.format(hb.timeout),
tags=hb.tags,
type='heartbeatAlert'
)
elif latency_exceeded:
alert = Alert(
resource=hb.origin,
event='HeartbeatSlow',
correlate=['HeartbeatFail', 'HeartbeatSlow', 'HeartbeatOK'],
group='System',
environment='Production',
service=['Alerta'],
severity='major',
value='{}ms'.format(latency),
text='Heartbeat took more than {}ms to be processed'.format(MAX_LATENCY),
tags=hb.tags,
type='heartbeatAlert'
)
else:
alert = Alert(
resource=hb.origin,
event='HeartbeatOK',
correlate=['HeartbeatFail', 'HeartbeatSlow', 'HeartbeatOK'],
group='System',
environment='Production',
service=['Alerta'],
severity='normal',
value='',
text='Heartbeat OK',
tags=hb.tags,
type='heartbeatAlert'
)
self.send(alert)