def test_process_unprocessed_events():
app = Comet()
app.register_parser('datastoretest', json)
app.register_parser('datastoretest2', json)
app.register_parser('datastoretest3', json)
app.set_config('datastoretest2', {})
specific_router = mock.Mock()
router = mock.Mock()
escalator = mock.Mock()
app.register_router('datastoretest2', func=specific_router)
app.register_router(func=router)
app.register_escalator(func=escalator)
check_user = '******'
already_processed_user = '******'
app.data_store.add_record(
EventRecord(id=1,
received_at=datetime.utcnow() - timedelta(days=5),
source_type='datastoretest',
owner=already_processed_user,
data={},
processed_at=datetime.utcnow() - timedelta(days=5),
fingerprint='f1'))
app.data_store.add_record(
EventRecord(id=2,
received_at=datetime.utcnow() - timedelta(days=4),
source_type='datastoretest',
owner=already_processed_user,
data={},
fingerprint='f1'))
app.data_store.add_record(
EventRecord(id=3,
received_at=datetime.utcnow() - timedelta(days=3),
source_type='datastoretest2',
owner=check_user,
data={},
fingerprint='f3'))
app.data_store.add_record(
EventRecord(id=4,
received_at=datetime.utcnow() - timedelta(days=3),
source_type='datastoretest3',
owner=check_user,
data={},
fingerprint='f4'))
app.process_unprocessed_events()
assert specific_router.call_count == 1
assert router.call_count == 2
assert router.call_args[0][2][0].owner == check_user
assert escalator.call_count == 3
def test_handle_non_addressed_events():
app = Comet()
app.register_parser('real_time_source', json)
app.register_parser('real_time_source2', json)
app.register_real_time_source('real_time_source')
app.register_real_time_source('real_time_source2')
app.set_config('real_time_source', {'alerts': {
'alert search name':
{
'escalate_cadence': timedelta(minutes=45),
'template': 'alerts_template'
}
}})
app.set_config('real_time_source2', {'alerts': {
'alert search name':
{
'escalate_cadence': timedelta(minutes=45),
'template': 'alerts_template'
}
}})
escalator = mock.Mock()
escalator2 = mock.Mock()
app.register_escalator('real_time_source', func=escalator)
app.register_escalator('real_time_source2', func=escalator2)
already_processed_user = '******'
# already processed real time event - needs escalation
app.data_store.add_record(
EventRecord(id=2,
received_at=datetime.utcnow() - timedelta(hours=1),
source_type='real_time_source',
owner=already_processed_user,
processed_at=datetime.utcnow() - timedelta(hours=1),
sent_at=datetime.utcnow() - timedelta(hours=1),
data={'search_name': 'alert search name',
'name': 'needs escalation'},
fingerprint='f2'))
# already processed real time event - still early for escalation
# the event sent 35 min ago.
app.data_store.add_record(
EventRecord(id=3,
received_at=datetime.utcnow() - timedelta(hours=1),
source_type='real_time_source2',
owner=already_processed_user,
processed_at=datetime.utcnow() - timedelta(minutes=35),
sent_at=datetime.utcnow() - timedelta(minutes=35),
data={'search_name': 'alert search name',
'name': 'doesnt need escalation'},
fingerprint='f3'))
app.handle_non_addressed_events()
assert escalator.call_count == 1
assert escalator2.call_count == 0
# Inputs
APP.register_input(
PubSubInput,
subscription_name=
'projects/comet-example-project/subscriptions/comet-example-subscription')
LOG.info('Registered inputs.')
# Parsers
APP.register_parser('forseti', ForsetiSchema)
APP.register_parser('detectify', DetectifySchema)
LOG.info('Registered parsers.')
# Override configuration
APP.set_config('forseti', {'wait_for_more': timedelta(minutes=2)})
# Hydrators
@APP.register_hydrator('forseti')
def hydrate_forseti(event):
"""Hydrator for forseti events, enriching them with owner, a fingerprint and metadata.
Args:
event (comet_core.app.EventContainer): the incoming event to hydrate
"""
msg = event.message
event.set_owner(f"{msg.get('project_owner')}@example.com")
event.set_fingerprint(
comet_event_fingerprint(data_dict=msg,
def test_process_unprocessed_events_digest_mode():
app = Comet()
app.register_parser("datastoretest", json)
app.register_parser("datastoretest2", json)
app.register_parser("datastoretest3", json)
app.register_parser("datastoretest4", json)
app.set_config("datastoretest2", {})
specific_router = mock.Mock()
router = mock.Mock()
escalator = mock.Mock()
app.register_router("datastoretest2", func=specific_router)
app.register_router(func=router)
app.register_escalator(func=escalator)
check_user = "******"
already_processed_user = "******"
app.data_store.add_record(
EventRecord(
id=1,
received_at=datetime.utcnow() - timedelta(days=5),
source_type="datastoretest",
owner=already_processed_user,
data={},
processed_at=datetime.utcnow() - timedelta(days=5),
fingerprint="f1",
)
)
app.data_store.add_record(
EventRecord(
id=2,
received_at=datetime.utcnow() - timedelta(days=4),
source_type="datastoretest",
owner=already_processed_user,
data={},
fingerprint="f1",
)
)
app.data_store.add_record(
EventRecord(
id=3,
received_at=datetime.utcnow() - timedelta(days=3),
source_type="datastoretest2",
owner=check_user,
data={},
fingerprint="f3",
)
)
app.data_store.add_record(
EventRecord(
id=4,
received_at=datetime.utcnow() - timedelta(days=3),
source_type="datastoretest3",
owner=check_user,
data={},
fingerprint="f4",
)
)
app.process_unprocessed_events()
# it is expected to have specific_router called once for the datastoretest2
assert specific_router.call_count == 1
# it is expected to have two calls of the generic router for the source_type datastoretest2 and datastoretest3
assert router.call_count == 2
# and the user must be check_user
assert router.call_args[0][2][0].owner == check_user
# due to the default escalation_time=10seconds, all three events (id=2,3,4) must be escalated
assert escalator.call_count == 3
app.data_store.add_record(
EventRecord(
id=5,
received_at=datetime.utcnow() - timedelta(days=2),
source_type="datastoretest",
owner=check_user,
data={},
fingerprint="f1",
)
)
app.process_unprocessed_events()
# f1 is expected to be processed, but not sent out
assert app.data_store.get_latest_event_with_fingerprint("f1").processed_at
assert not app.data_store.get_latest_event_with_fingerprint("f1").sent_at
def test_process_unprocessed_events_non_digest_mode():
app = Comet()
app.register_parser("datastoretest4", json)
check_user = "******"
router = mock.Mock()
escalator = mock.Mock()
app.register_router(func=router)
app.register_escalator(func=escalator)
app.set_config("datastoretest4", {"communication_digest_mode": False, "new_threshold": timedelta(days=14)})
app.data_store.add_record(
EventRecord(
id=6,
received_at=datetime.utcnow() - timedelta(days=8),
source_type="datastoretest4",
sent_at=datetime.utcnow() - timedelta(days=8),
processed_at=datetime.utcnow() - timedelta(days=8),
owner=check_user,
data={},
fingerprint="f5",
)
)
app.data_store.add_record(
EventRecord(
id=7,
received_at=datetime.utcnow() - timedelta(days=2),
source_type="datastoretest4",
owner=check_user,
data={},
fingerprint="f5",
)
)
app.data_store.add_record(
EventRecord(
id=8,
received_at=datetime.utcnow(),
source_type="datastoretest4",
owner=check_user,
data={},
fingerprint="f6",
)
)
app.data_store.add_record(
EventRecord(
id=9,
received_at=datetime.utcnow() - timedelta(days=2),
source_type="datastoretest4",
sent_at=datetime.utcnow() - timedelta(days=2),
processed_at=datetime.utcnow() - timedelta(days=2),
owner=check_user,
data={},
fingerprint="f7",
)
)
app.data_store.add_record(
EventRecord(
id=10,
received_at=datetime.utcnow(),
source_type="datastoretest4",
owner=check_user,
data={},
fingerprint="f7",
)
)
# f5 is expected to be reminded
# f6 is expected to be new and sent as well
# f7 is NOT expected to be reminded
before_calling = router.call_count
app.process_unprocessed_events()
assert app.data_store.get_latest_event_with_fingerprint("f5").processed_at
assert app.data_store.get_latest_event_with_fingerprint("f6").processed_at
assert app.data_store.get_latest_event_with_fingerprint("f7").processed_at
assert router.call_count == before_calling + 1
sent_fingerprints = [e.fingerprint for e in router.call_args[0][2]]
assert "f5" in sent_fingerprints
assert "f6" in sent_fingerprints
assert "f7" not in sent_fingerprints