def handle_gc_trigger_incidents(self):
"""
GCs incidents.
First, find all expired incidents.
Second, use expired incidents to delete expired gridfs data.
Third, remove expired incidents.
"""
self.debug("gc_trigger incidents")
now = timestamp_s()
incidents = self.mongo_client.incidents_collection.find(
{"expire_at": {
"$lt": now
}},
projection={
"expire_at": True,
"gridfs_filename": True
})
filenames = list(
map(lambda incident: incident["gridfs_filename"], incidents))
for filename in filenames:
self.mongo_client.delete_gridfs(filename)
delete_result = self.mongo_client.incidents_collection.delete_many(
{"expire_at": {
"$lt": now
}})
self.produce(
Routes.gc_stat,
util_pb2.build_gc_stat(self.NAME, mauka_pb2.INCIDENTS,
delete_result.deleted_count))
self.debug(
"Garbage collected %d incidents and associated gridfs data" %
delete_result.deleted_count)
def test_which_message_oneof(self):
payload = pb_util.build_payload("", 0, "", pb_util.mauka_pb2.MEASUREMENTS, [], 0, 0)
self.assertEqual(pb_util.which_message_oneof(payload), pb_util.PAYLOAD)
heartbeat = pb_util.build_heartbeat("", 0, 0, "", "")
self.assertEqual(pb_util.which_message_oneof(heartbeat), pb_util.HEARTBEAT)
makai_event = pb_util.build_makai_event("", 0)
self.assertEqual(pb_util.which_message_oneof(makai_event), pb_util.MAKAI_EVENT)
measurement = pb_util.build_measurement("", "", 0, 0, 0, 0)
self.assertEqual(pb_util.which_message_oneof(measurement), pb_util.MEASUREMENT)
makai_trigger = pb_util.build_makai_trigger("", 0, 0, "", 0, "")
self.assertEqual(pb_util.which_message_oneof(makai_trigger), pb_util.MAKAI_TRIGGER)
laha_ttl = pb_util.build_ttl("", "", 0)
self.assertEqual(pb_util.which_message_oneof(laha_ttl), pb_util.LAHA)
laha_gc_trigger = pb_util.build_gc_trigger("", [])
self.assertEqual(pb_util.which_message_oneof(laha_gc_trigger), pb_util.LAHA)
laha_gc_update = pb_util.build_gc_update("", pb_util.mauka_pb2.MEASUREMENTS, 0)
self.assertEqual(pb_util.which_message_oneof(laha_gc_update), pb_util.LAHA)
laha_gc_stat = pb_util.build_gc_stat("", pb_util.mauka_pb2.MEASUREMENTS, 0)
self.assertEqual(pb_util.which_message_oneof(laha_gc_stat), pb_util.LAHA)
trigger_request = pb_util.build_trigger_request("", 0, 0, [], 0)
self.assertEqual(pb_util.which_message_oneof(trigger_request), pb_util.TRIGGER_REQUEST)
triggered_event = pb_util.build_triggered_event("", [], 0, "", 0, 0)
self.assertEqual(pb_util.which_message_oneof(triggered_event), pb_util.TRIGGERED_EVENT)
threshold_optimization_request = pb_util.build_threshold_optimization_request("")
self.assertEqual(pb_util.which_message_oneof(threshold_optimization_request), pb_util.THRESHOLD_OPTIMIZATION_REQUEST)
box_optimization_request = pb_util.build_box_optimization_request("", [], 0)
self.assertEqual(pb_util.which_message_oneof(box_optimization_request), pb_util.BOX_OPTIMIZATION_REQUEST)
def test_build_gc_stat(self):
mauka_message = pb_util.build_gc_stat("test",
pb_util.mauka_pb2.TRENDS,
1)
self.assertTrue(isinstance(mauka_message, pb_util.mauka_pb2.MaukaMessage))
self.assertTrue(pb_util.is_gc_stat(mauka_message))
self.assertEqual(mauka_message.source, "test")
self.assertEqual(mauka_message.laha.gc_stat.gc_domain, pb_util.mauka_pb2.TRENDS)
self.assertEqual(mauka_message.laha.gc_stat.gc_cnt, 1)
def handle_gc_trigger_trends(self):
"""
GCs trends by removing trends older than their expire_at field.
"""
self.debug("gc_trigger trends")
now = timestamp_s()
delete_result = self.mongo_client.trends_collection.delete_many(
{"expire_at": {
"$lt": now
}})
self.produce(
Routes.gc_stat,
util_pb2.build_gc_stat(self.NAME, mauka_pb2.TRENDS,
delete_result.deleted_count))
self.debug("Garbage collected %d trends" % delete_result.deleted_count)
def test_is_laha_and_related(self):
laha_ttl = pb_util.build_ttl("", "", 0)
laha_gc_trigger = pb_util.build_gc_trigger("", [])
laha_gc_update = pb_util.build_gc_update("", pb_util.mauka_pb2.MEASUREMENTS, 0)
laha_gc_stat = pb_util.build_gc_stat("", pb_util.mauka_pb2.MEASUREMENTS, 0)
laha_messages = [laha_ttl, laha_gc_trigger, laha_gc_update, laha_gc_stat]
for message in laha_messages:
self.assertTrue(pb_util.is_laha(message))
self.assertTrue(pb_util.is_ttl(laha_ttl))
self.assertFalse(pb_util.is_ttl(laha_gc_stat))
self.assertTrue(pb_util.is_gc_trigger(laha_gc_trigger))
self.assertTrue(pb_util.is_gc_update(laha_gc_update))
self.assertTrue(pb_util.is_gc_stat(laha_gc_stat))
def handle_gc_trigger_measurements(self):
"""
GCs measurements by removing measurements older than their expire_at field.
"""
self.debug("gc_trigger measurements")
now = timestamp_s()
delete_result = self.mongo_client.measurements_collection.delete_many(
{"expire_at": {
"$lt": now
}})
self.produce(
Routes.gc_stat,
util_pb2.build_gc_stat(self.NAME, mauka_pb2.MEASUREMENTS,
delete_result.deleted_count))
self.debug("Garbage collected %d measurements" %
delete_result.deleted_count)
def handle_gc_trigger_events(self):
"""
GCs events.
First, find all events whose expire_at field is older than now.
Second, find corresponding box_events.
Third, from box_events, delete expired gridfs files
Fourth, delete all expired box_events
Fifth, delete all expired events
"""
self.debug("gc_trigger events")
now = timestamp_s()
events = self.mongo_client.events_collection.find(
{"expire_at": {
"$lt": now
}},
projection={
"_id": True,
"expire_at": True,
"event_id": True,
"boxes_received": True
})
# Find corresponding box events
box_events = []
for event in events:
for box_triggered in event["boxes_received"]:
box_event = self.mongo_client.box_events_collection.find_one(
{
"event_id": event["event_id"],
"box_id": box_triggered
},
projection={
"_id": True,
"event_id": True,
"box_id": True,
"data_fs_filename": True
})
if box_event is not None:
box_events.append(box_event)
# Cleanup gridfs files and box_events
for box_event in box_events:
self.mongo_client.delete_gridfs(box_event["data_fs_filename"])
self.mongo_client.box_events_collection.delete_one(
{"_id": box_event["_id"]})
self.debug(
"Garbage collected %d box_events and corresponding gridfs data" %
len(box_events))
# Cleanup events
delete_result = self.mongo_client.events_collection.delete_many(
{"expire_at": {
"$lt": now
}})
self.produce(
Routes.gc_stat,
util_pb2.build_gc_stat(self.NAME, mauka_pb2.EVENTS,
delete_result.deleted_count))
self.debug("Garbage collected %d events" % delete_result.deleted_count)