def dispatch(self):
"""Find and dispatch all pending alerts to the alert processor."""
# To reduce the API calls to Dynamo, batch all additions and deletions until the end.
merged_alerts = [] # List of newly created merge alerts
alerts_to_delete = [] # List of alerts which can be deleted
for rule_name in self.table.rule_names_generator():
merge_enabled_alerts = []
for alert in self._alert_generator(rule_name):
if alert.remaining_outputs:
# If an alert still has pending outputs, it needs to be sent immediately.
# For example, all alerts are sent to the default firehose now even if they will
# later be merged when sending to other outputs.
self._dispatch_alert(alert)
elif alert.merge_enabled:
# This alert has finished sending to non-merged outputs; it is now a candidate
# for alert merging.
merge_enabled_alerts.append(alert)
else:
# This alert has sent successfully but doesn't need to be merged.
# It should have been deleted by the alert processor, but we can do it now.
alerts_to_delete.append(alert)
for group in self._merge_groups(merge_enabled_alerts):
# Create a new merged Alert.
new_alert = Alert.merge(group.alerts)
LOGGER.info('Merged %d alerts into a new alert with ID %s',
len(group.alerts), new_alert.alert_id)
merged_alerts.append(new_alert)
# Since we already guaranteed that the original alerts have sent to the unmerged
# outputs (e.g. default firehose), they can be safely marked for deletion.
alerts_to_delete.extend(group.alerts)
if merged_alerts:
# Add new merged alerts to the alerts table and send them to the alert processor.
self.table.add_alerts(merged_alerts)
for alert in merged_alerts:
self._dispatch_alert(alert)
if alerts_to_delete:
self.table.delete_alerts([(alert.rule_name, alert.alert_id)
for alert in alerts_to_delete])
def test_merge_nested(self):
"""Alert Class - Merge - Merge with Nested Keys"""
record1 = {
'NumMatchedRules': 1,
'FileInfo': {
'Deleted': None,
'Nested': [1, 2, 'three']
},
'MatchedRules': {
'Rule1': 'MatchedStrings'
}
}
alert1 = Alert('RuleName',
record1, {'slack:channel'},
created=datetime(year=2000, month=1, day=1),
merge_by_keys=['Nested'],
merge_window=timedelta(minutes=5))
record2 = {
'NumMatchedRules': 2,
'FileInfo': {
'Deleted': None,
'Nested': [1, 2, 'three']
},
'MatchedRules': {
'Rule1': 'MatchedStrings'
}
}
alert2 = Alert('RuleName',
record2, {'slack:channel'},
created=datetime(year=2000, month=1, day=2),
merge_by_keys=['Nested'],
merge_window=timedelta(minutes=5))
record3 = {
'MatchedRules': {
'Rule1': 'MatchedStrings'
},
'Nested': [1, 2,
'three'] # This is in a different place in the record
}
alert3 = Alert('RuleName',
record3, {'slack:channel'},
created=datetime(year=2000, month=1, day=3),
merge_by_keys=['Nested'],
merge_window=timedelta(minutes=5))
merged = Alert.merge([alert1, alert2, alert3])
expected_record = {
'AlertCount': 3,
'AlertTimeFirst': '2000-01-01T00:00:00.000000Z',
'AlertTimeLast': '2000-01-03T00:00:00.000000Z',
'MergedBy': {
'Nested': [1, 2, 'three']
},
'OtherCommonKeys': {
'MatchedRules': {
'Rule1': 'MatchedStrings'
}
},
'ValueDiffs': {
'2000-01-01T00:00:00.000000Z': {
'NumMatchedRules': 1,
'FileInfo': {
'Deleted': None
}
},
'2000-01-02T00:00:00.000000Z': {
'NumMatchedRules': 2,
'FileInfo': {
'Deleted': None
}
},
'2000-01-03T00:00:00.000000Z': {}
}
}
assert_equal(expected_record, merged.record)
def test_merge(self):
"""Alert Class - Merge - Create Merged Alert"""
# Example based on a CarbonBlack log
record1 = {
'alliance_data_virustotal': [],
'alliance_link_virustotal': '',
'alliance_score_virustotal': 0,
'cmdline': 'whoami',
'comms_ip': '1.2.3.4',
'hostname': 'my-computer-name',
'path': '/usr/bin/whoami',
'streamalert:ioc': {
'hello': 'world'
},
'timestamp': 1234.5678,
'username': '******'
}
alert1 = Alert('RuleName',
record1, {'aws-sns:topic'},
created=datetime(year=2000, month=1, day=1),
merge_by_keys=['hostname', 'username'],
merge_window=timedelta(minutes=5))
# Second alert has slightly different record and different outputs
record2 = copy.deepcopy(record1)
record2['streamalert:ioc'] = {'goodbye': 'world'}
record2['timestamp'] = 9999
alert2 = Alert('RuleName',
record2, {'slack:channel'},
created=datetime(year=2000, month=1, day=2),
merge_by_keys=['hostname', 'username'],
merge_window=timedelta(minutes=5))
merged = Alert.merge([alert1, alert2])
assert_is_instance(merged, Alert)
assert_equal({'slack:channel'},
merged.outputs) # Most recent outputs were used
expected_record = {
'AlertCount': 2,
'AlertTimeFirst': '2000-01-01T00:00:00.000000Z',
'AlertTimeLast': '2000-01-02T00:00:00.000000Z',
'MergedBy': {
'hostname': 'my-computer-name',
'username': '******'
},
'OtherCommonKeys': {
'alliance_data_virustotal': [],
'alliance_link_virustotal': '',
'alliance_score_virustotal': 0,
'cmdline': 'whoami',
'comms_ip': '1.2.3.4',
'path': '/usr/bin/whoami',
},
'ValueDiffs': {
'2000-01-01T00:00:00.000000Z': {
'streamalert:ioc': {
'hello': 'world'
},
'timestamp': 1234.5678
},
'2000-01-02T00:00:00.000000Z': {
'streamalert:ioc': {
'goodbye': 'world'
},
'timestamp': 9999
}
}
}
assert_equal(expected_record, merged.record)