def test_not_filter(self):
notfilter = NotFilter([
InFilter(FieldValue(LogField.SERVICE), [ServiceValue('UDP/53', 'TCP/80')])])
f = {'type': 'not',
'value': {'left': {'id': 27, 'type': 'field'},
'right': [{'type': 'service', 'value': 'UDP/53'},
{'type': 'service', 'value': 'TCP/80'}],
'type': 'in'}}
self.assertDictEqual(notfilter.filter, f)
query = LogQuery()
query.add_not_filter([
InFilter(FieldValue(LogField.SERVICE), [ServiceValue('UDP/53', 'TCP/80')])])
query_filter = query.request['query']['filter']
self.assertDictEqual(f, query_filter)
notfilter.update_filter([
InFilter(FieldValue(LogField.HTTPREQUESTHOST),[StringValue('play.googleapis.com')])])
d = {'type': 'not',
'value': {'left': {'id': 1586, 'type': 'field'},
'right': [{'type': 'string', 'value': 'play.googleapis.com'}],
'type': 'in'}}
self.assertDictEqual(d, notfilter.filter)
query.update_filter(
NotFilter([InFilter(FieldValue(LogField.HTTPREQUESTHOST),[StringValue('play.googleapis.com')])])
)
query_filter = query.request['query']['filter']
self.assertDictEqual(d, query_filter)
def name_with_fields(self):
query = LogQuery(fetch_size=100)
query.add_in_filter(
FieldValue(LogField.SRC), [IPValue('192.168.4.84')])
query.format.field_format('name')
logquery = {
"format": {
"type": "texts",
"field_format": "name",
"resolving": {"senders": True}
},
"query": {
"type":"stored",
"end_ms": 0,
"start_ms": 0,
"filter": {
"type":"in",
"left": {
"type": "field",
"id": LogField.SRC},
"right":[{
"type": "ip",
"value": "192.168.4.84"}]}
},
"fetch":{"quantity":100, 'backwards': True}
}
self.assertDictEqual(query.request, logquery)
def test_infilter(self):
# Naked INFilter
infilter = InFilter(FieldValue(LogField.SRC), [IPValue('172.18.1.1', '172.18.1.2')])
d = {'left': {'id': 7, 'type': 'field'},
'right': [{'type': 'ip', 'value': '172.18.1.1'},
{'type': 'ip', 'value': '172.18.1.2'}],
'type': 'in'}
self.assertDictEqual(infilter.filter, d)
# Add to query using query method
query = LogQuery()
query.add_in_filter(
FieldValue(LogField.SRC), [IPValue('172.18.1.1', '172.18.1.2')])
query_filter = query.request['query']['filter']
self.assertDictEqual(d, query_filter)
# Update the original filter, validate update
infilter.update_filter(FieldValue('Src'), [IPValue('1.1.1.1')])
d = {'left': {'name': 'Src', 'type': 'field'},
'right': [{'type': 'ip', 'value': '1.1.1.1'}],
'type': 'in'}
# Update filter on query
self.assertDictEqual(infilter.filter, d)
query.update_filter(InFilter(FieldValue('Src'), [IPValue('1.1.1.1')]))
query_filter = query.request['query']['filter']
self.assertDictEqual(d, query_filter)
def test_translated(self):
trans_filter = TranslatedFilter()
trans_filter.within_ipv4_network('$Dst', ['192.168.4.0/24'])
f = {'type': 'translated', 'value': '$Dst IN union(ipv4_net("192.168.4.0/24"))'}
self.assertDictEqual(f, trans_filter.filter)
trans_filter.within_ipv4_range('$Src', ['1.1.1.1-192.168.1.254'])
f = {'type': 'translated',
'value': '$Src IN range(ipv4("1.1.1.1"),ipv4("192.168.1.254"))'}
self.assertDictEqual(f, trans_filter.filter)
trans_filter.exact_ipv4_match('$Dst', ['1.1.1.1'])
f = {'type': 'translated', 'value': '$Dst == ipv4("1.1.1.1")'}
self.assertDictEqual(f, trans_filter.filter)
trans_filter.exact_ipv4_match('$Src', ['172.18.1.152', '192.168.4.84'])
f = {'type': 'translated',
'value': '$Src IN union(ipv4("172.18.1.152"),ipv4("192.168.4.84"))'}
self.assertDictEqual(f, trans_filter.filter)
qt = LogQuery()
tr = qt.add_translated_filter()
tr.exact_ipv4_match('$Src', ['172.18.1.152', '192.168.4.84'])
query_filter = qt.request['query']['filter']
self.assertDictEqual(query_filter, f)
def test_defined(self):
query = LogQuery()
query.add_defined_filter(FieldValue(LogField.ACTION))
f = {"type": "defined", "value": {"id": 14, "type": "field"}}
query_filter = query.request["query"]["filter"]
self.assertDictEqual(f, query_filter)
def test_translated(self):
trans_filter = TranslatedFilter()
trans_filter.within_ipv4_network("$Dst", ["192.168.4.0/24"])
f = {
"type": "translated",
"value": '$Dst IN union(ipv4_net("192.168.4.0/24"))'
}
self.assertDictEqual(f, trans_filter.filter)
trans_filter.within_ipv4_range("$Src", ["1.1.1.1-192.168.1.254"])
f = {
"type": "translated",
"value": '$Src IN range(ipv4("1.1.1.1"),ipv4("192.168.1.254"))'
}
self.assertDictEqual(f, trans_filter.filter)
trans_filter.exact_ipv4_match("$Dst", ["1.1.1.1"])
f = {"type": "translated", "value": '$Dst == ipv4("1.1.1.1")'}
self.assertDictEqual(f, trans_filter.filter)
trans_filter.exact_ipv4_match("$Src", ["172.18.1.152", "192.168.4.84"])
f = {
"type": "translated",
"value":
'$Src IN union(ipv4("172.18.1.152"),ipv4("192.168.4.84"))',
}
self.assertDictEqual(f, trans_filter.filter)
qt = LogQuery()
tr = qt.add_translated_filter()
tr.exact_ipv4_match("$Src", ["172.18.1.152", "192.168.4.84"])
query_filter = qt.request["query"]["filter"]
self.assertDictEqual(query_filter, f)
def test_defined(self):
query = LogQuery()
query.add_defined_filter(FieldValue(LogField.ACTION))
f = {'type': 'defined', 'value': {'id': 14, 'type': 'field'}}
query_filter = query.request['query']['filter']
self.assertDictEqual(f, query_filter)
def test_translated(self):
trans_filter = TranslatedFilter()
trans_filter.within_ipv4_network('$Dst', ['192.168.4.0/24'])
f = {
'type': 'translated',
'value': '$Dst IN union(ipv4_net("192.168.4.0/24"))'
}
self.assertDictEqual(f, trans_filter.filter)
trans_filter.within_ipv4_range('$Src', ['1.1.1.1-192.168.1.254'])
f = {
'type': 'translated',
'value': '$Src IN range(ipv4("1.1.1.1"),ipv4("192.168.1.254"))'
}
self.assertDictEqual(f, trans_filter.filter)
trans_filter.exact_ipv4_match('$Dst', ['1.1.1.1'])
f = {'type': 'translated', 'value': '$Dst == ipv4("1.1.1.1")'}
self.assertDictEqual(f, trans_filter.filter)
trans_filter.exact_ipv4_match('$Src', ['172.18.1.152', '192.168.4.84'])
f = {
'type': 'translated',
'value': '$Src IN union(ipv4("172.18.1.152"),ipv4("192.168.4.84"))'
}
self.assertDictEqual(f, trans_filter.filter)
qt = LogQuery()
tr = qt.add_translated_filter()
tr.exact_ipv4_match('$Src', ['172.18.1.152', '192.168.4.84'])
query_filter = qt.request['query']['filter']
self.assertDictEqual(query_filter, f)
def test_defined(self):
query = LogQuery()
query.add_defined_filter(FieldValue(LogField.ACTION))
f = {'type': 'defined',
'value': {'id': 14, 'type': 'field'}}
query_filter = query.request['query']['filter']
self.assertDictEqual(f, query_filter)
def test_and_filter(self):
andfilter = AndFilter([
InFilter(FieldValue(LogField.SRC), [IPValue('192.168.4.84')]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue('TCP/80')])])
f = {'type': 'and',
'values': [{'left': {'id': 7, 'type': 'field'},
'right': [{'type': 'ip', 'value': '192.168.4.84'}],
'type': 'in'},
{'left': {'id': 27, 'type': 'field'},
'right': [{'type': 'service', 'value': 'TCP/80'}],
'type': 'in'}]}
self.assertDictEqual(andfilter.filter, f)
query = LogQuery()
query.add_and_filter([
InFilter(FieldValue(LogField.SRC), [IPValue('192.168.4.84')]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue('TCP/80')])])
query_filter = query.request['query']['filter']
self.assertDictEqual(f, query_filter)
# Update the original filter
andfilter.update_filter(
[InFilter(FieldValue(LogField.DST), [IPValue('1.1.1.1')]),
InFilter(FieldValue(LogField.ACTION), [ConstantValue(Actions.DISCARD, Actions.BLOCK)])])
d = {'type': 'and',
'values': [{'left': {'id': 8, 'type': 'field'},
'right': [{'type': 'ip', 'value': '1.1.1.1'}],
'type': 'in'},
{'left': {'id': 14, 'type': 'field'},
'right': [{'type': 'constant', 'value': 0},
{'type': 'constant', 'value': 13}],
'type': 'in'}]}
self.assertDictEqual(andfilter.filter, d)
query.update_filter(
AndFilter([
InFilter(FieldValue(LogField.DST), [IPValue('1.1.1.1')]),
InFilter(FieldValue(LogField.ACTION), [ConstantValue(Actions.DISCARD, Actions.BLOCK)])
])
)
query_filter = query.request['query']['filter']
self.assertDictEqual(d, query_filter)
def test_log_query_params(self):
query = LogQuery(fetch_size=50)
# Default format is TextFormat
self.assertIsInstance(query.format, TextFormat)
orig = {
"fetch": {
"backwards": True,
"quantity": 50
},
"format": {
"field_format": "pretty",
"resolving": {
"senders": True
},
"type": "texts"
},
"query": {
"end_ms": 0,
"start_ms": 0,
"type": "stored"
},
}
self.assertDictEqual(orig, query.request)
# Add a timezone
query.format.timezone("CST")
# Check that 'resolving' was modified on the format
resolving = {
"resolving": {
"timezone": "CST",
"time_show_zone": True,
"senders": True
},
"type": "texts",
"field_format": "pretty",
}
self.assertDictEqual(resolving, query.format.data)
# Test that set_resolving works properly
query.format.set_resolving(timezone="PST")
self.assertEqual(query.format.data["resolving"]["timezone"], "PST")
# Change field format from 'pretty' to 'name'
query.format.field_format("name")
self.assertEqual(query.format.data["field_format"], "name")
# Add field names to the query..
query.format.field_names(["Src", "Dst"])
for name in query.format.data["field_names"]:
self.assertIn(name, ["Src", "Dst"])
# Add field IDs to the query
query.format.field_ids([LogField.SRC, LogField.IPSAPPID])
for _id in query.format.data["field_ids"]:
self.assertIn(_id, [LogField.SRC, LogField.IPSAPPID])
def test_log_query_params(self):
query = LogQuery(fetch_size=50)
# Default format is TextFormat
self.assertIsInstance(query.format, TextFormat)
orig = {
'fetch': {
'backwards': True,
'quantity': 50
},
'format': {
'field_format': 'pretty',
'resolving': {
'senders': True
},
'type': 'texts'
},
'query': {
'end_ms': 0,
'start_ms': 0,
'type': 'stored'
}
}
self.assertDictEqual(orig, query.request)
# Add a timezone
query.format.timezone('CST')
# Check that 'resolving' was modified on the format
resolving = {
'resolving': {
'timezone': 'CST',
'time_show_zone': True,
'senders': True
},
'type': 'texts',
'field_format': 'pretty'
}
self.assertDictEqual(resolving, query.format.data)
# Test that set_resolving works properly
query.format.set_resolving(timezone='PST')
self.assertEqual(query.format.data['resolving']['timezone'], 'PST')
# Change field format from 'pretty' to 'name'
query.format.field_format('name')
self.assertEqual(query.format.data['field_format'], 'name')
# Add field names to the query..
query.format.field_names(['Src', 'Dst'])
for name in query.format.data['field_names']:
self.assertIn(name, ['Src', 'Dst'])
# Add field IDs to the query
query.format.field_ids([LogField.SRC, LogField.IPSAPPID])
for _id in query.format.data['field_ids']:
self.assertIn(_id, [LogField.SRC, LogField.IPSAPPID])
def test_or_filter(self):
orfilter = OrFilter([
InFilter(FieldValue(LogField.SRC), [IPValue('192.168.4.84')]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue('TCP/80')])])
f = {'type': 'or',
'values': [{'left': {'id': 7, 'type': 'field'},
'right': [{'type': 'ip', 'value': '192.168.4.84'}],
'type': 'in'},
{'left': {'id': 27, 'type': 'field'},
'right': [{'type': 'service', 'value': 'TCP/80'}],
'type': 'in'}]}
self.assertDictEqual(orfilter.filter, f)
query = LogQuery()
query.add_or_filter([
InFilter(FieldValue(LogField.SRC), [IPValue('192.168.4.84')]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue('TCP/80')])])
query_filter = query.request['query']['filter']
self.assertDictEqual(f, query_filter)
orfilter.update_filter([
InFilter(FieldValue(LogField.DST), [IPValue('1.1.1.1')]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue('TCP/443')])])
d = {'type': 'or',
'values': [{'left': {'id': 8, 'type': 'field'},
'right': [{'type': 'ip', 'value': '1.1.1.1'}],
'type': 'in'},
{'left': {'id': 27, 'type': 'field'},
'right': [{'type': 'service', 'value': 'TCP/443'}],
'type': 'in'}]}
self.assertDictEqual(d, orfilter.filter)
query.update_filter(
OrFilter([
InFilter(FieldValue(LogField.DST), [IPValue('1.1.1.1')]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue('TCP/443')])])
)
query_filter = query.request['query']['filter']
self.assertDictEqual(d, query_filter)
def name_with_fields(self):
query = LogQuery(fetch_size=100)
query.add_in_filter(FieldValue(LogField.SRC),
[IPValue("192.168.4.84")])
query.format.field_format("name")
logquery = {
"format": {
"type": "texts",
"field_format": "name",
"resolving": {
"senders": True
}
},
"query": {
"type": "stored",
"end_ms": 0,
"start_ms": 0,
"filter": {
"type": "in",
"left": {
"type": "field",
"id": LogField.SRC
},
"right": [{
"type": "ip",
"value": "192.168.4.84"
}],
},
},
"fetch": {
"quantity": 100,
"backwards": True
},
}
self.assertDictEqual(query.request, logquery)
def test_query_time_format(self):
query = LogQuery()
self.assertEqual(query.time_range.start_time, 0)
self.assertEqual(query.time_range.end_time, 0)
# Make sure time range is changed
query.time_range.last_five_minutes()
fivemin_start = query.time_range.start_time
fivemin_end = query.time_range.end_time
self.assertNotEqual(fivemin_start, 0)
self.assertNotEqual(fivemin_end, 0)
# Change again
query.time_range.last_fifteen_minutes()
fifteen_min_start = query.time_range.start_time
fifteen_min_end = query.time_range.end_time
self.assertNotEqual(fifteen_min_start, fivemin_start)
self.assertNotEqual(fifteen_min_start, fivemin_end)
from datetime import datetime
# Time conversions
dt = datetime(2017, 9, 2, 6, 25, 30, 0)
ms = datetime_to_ms(dt)
self.assertTrue(dt == datetime_from_ms(ms))
from pprint import pprint
tf = TimeFormat()
def get_time_diff():
return datetime_from_ms(tf.data.get('end_ms')) - \
datetime_from_ms(tf.data.get('start_ms'))
tf.last_five_minutes()
self.assertEqual(get_time_diff().seconds, 300) # 300 seconds = 5 min
tf.last_fifteen_minutes()
self.assertEqual(get_time_diff().seconds, 900)
tf.last_thirty_minutes()
self.assertEqual(get_time_diff().seconds, 1800)
tf.last_hour()
self.assertEqual(get_time_diff().seconds, 3600)
tf.last_day()
self.assertEqual(get_time_diff().days, 1)
tf.last_week()
self.assertEqual(get_time_diff().days, 7)
def test_infilter(self):
# Naked INFilter
infilter = InFilter(FieldValue(LogField.SRC),
[IPValue("172.18.1.1", "172.18.1.2")])
d = {
"left": {
"id": 7,
"type": "field"
},
"right": [{
"type": "ip",
"value": "172.18.1.1"
}, {
"type": "ip",
"value": "172.18.1.2"
}],
"type":
"in",
}
self.assertDictEqual(infilter.filter, d)
# Add to query using query method
query = LogQuery()
query.add_in_filter(FieldValue(LogField.SRC),
[IPValue("172.18.1.1", "172.18.1.2")])
query_filter = query.request["query"]["filter"]
self.assertDictEqual(d, query_filter)
# Update the original filter, validate update
infilter.update_filter(FieldValue("Src"), [IPValue("1.1.1.1")])
d = {
"left": {
"name": "Src",
"type": "field"
},
"right": [{
"type": "ip",
"value": "1.1.1.1"
}],
"type": "in",
}
# Update filter on query
self.assertDictEqual(infilter.filter, d)
query.update_filter(InFilter(FieldValue("Src"), [IPValue("1.1.1.1")]))
query_filter = query.request["query"]["filter"]
self.assertDictEqual(d, query_filter)
def test_infilter(self):
# Naked INFilter
infilter = InFilter(FieldValue(LogField.SRC),
[IPValue('172.18.1.1', '172.18.1.2')])
d = {
'left': {
'id': 7,
'type': 'field'
},
'right': [{
'type': 'ip',
'value': '172.18.1.1'
}, {
'type': 'ip',
'value': '172.18.1.2'
}],
'type':
'in'
}
self.assertDictEqual(infilter.filter, d)
# Add to query using query method
query = LogQuery()
query.add_in_filter(FieldValue(LogField.SRC),
[IPValue('172.18.1.1', '172.18.1.2')])
query_filter = query.request['query']['filter']
self.assertDictEqual(d, query_filter)
# Update the original filter, validate update
infilter.update_filter(FieldValue('Src'), [IPValue('1.1.1.1')])
d = {
'left': {
'name': 'Src',
'type': 'field'
},
'right': [{
'type': 'ip',
'value': '1.1.1.1'
}],
'type': 'in'
}
# Update filter on query
self.assertDictEqual(infilter.filter, d)
query.update_filter(InFilter(FieldValue('Src'), [IPValue('1.1.1.1')]))
query_filter = query.request['query']['filter']
self.assertDictEqual(d, query_filter)
def test_or_filter(self):
orfilter = OrFilter([
InFilter(FieldValue(LogField.SRC), [IPValue('192.168.4.84')]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue('TCP/80')])
])
f = {
'type':
'or',
'values': [{
'left': {
'id': 7,
'type': 'field'
},
'right': [{
'type': 'ip',
'value': '192.168.4.84'
}],
'type': 'in'
}, {
'left': {
'id': 27,
'type': 'field'
},
'right': [{
'type': 'service',
'value': 'TCP/80'
}],
'type': 'in'
}]
}
self.assertDictEqual(orfilter.filter, f)
query = LogQuery()
query.add_or_filter([
InFilter(FieldValue(LogField.SRC), [IPValue('192.168.4.84')]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue('TCP/80')])
])
query_filter = query.request['query']['filter']
self.assertDictEqual(f, query_filter)
orfilter.update_filter([
InFilter(FieldValue(LogField.DST), [IPValue('1.1.1.1')]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue('TCP/443')])
])
d = {
'type':
'or',
'values': [{
'left': {
'id': 8,
'type': 'field'
},
'right': [{
'type': 'ip',
'value': '1.1.1.1'
}],
'type': 'in'
}, {
'left': {
'id': 27,
'type': 'field'
},
'right': [{
'type': 'service',
'value': 'TCP/443'
}],
'type': 'in'
}]
}
self.assertDictEqual(d, orfilter.filter)
query.update_filter(
OrFilter([
InFilter(FieldValue(LogField.DST), [IPValue('1.1.1.1')]),
InFilter(FieldValue(LogField.SERVICE),
[ServiceValue('TCP/443')])
]))
query_filter = query.request['query']['filter']
self.assertDictEqual(d, query_filter)
def test_custom_formats_for_query(self):
dt = DetailedFormat(timezone='CST',
time_show_zone=True,
ip_elements=True,
ip_locations=True)
# Add detailed format to a query
query = LogQuery(format=dt)
d = {
'fetch': {
'backwards': True
},
'format': {
'field_format': 'pretty',
'resolving': {
'ip_elements': True,
'ip_locations': True,
'senders': True,
'time_show_zone': True,
'timezone': 'CST'
},
'type': 'detailed'
},
'query': {
'end_ms': 0,
'start_ms': 0,
'type': 'stored'
}
}
self.assertDictEqual(d, query.request)
# Change the format for a query
rf = RawFormat(field_format='name')
rf.field_ids([LogField.SRC, LogField.DST])
query.update_format(rf)
request = {
'fetch': {
'backwards': True
},
'format': {
'field_format': 'name',
'field_ids': [7, 8],
'type': 'raw'
},
'query': {
'end_ms': 0,
'start_ms': 0,
'type': 'stored'
}
}
self.assertDictEqual(request, query.request)
# Start with fresh Query and add a combined filter. A combined filter
# mixes individual formats together and when data is returned it's
# dict key will be the mapping to the format
text = TextFormat()
text.field_ids([LogField.TIMESTAMP])
detailed = DetailedFormat()
detailed.field_ids([LogField.SRC, LogField.DST])
combined = CombinedFormat(tformat=text, dformat=detailed)
query = LogQuery(fetch_size=1)
query.update_format(combined)
d = {
'fetch': {
'backwards': True,
'quantity': 1
},
'format': {
'formats': {
'dformat': {
'field_format': 'pretty',
'field_ids': [7, 8],
'resolving': {
'senders': True
},
'type': 'detailed'
},
'tformat': {
'field_format': 'pretty',
'field_ids': [1],
'resolving': {
'senders': True
},
'type': 'texts'
}
},
'type': 'combined'
},
'query': {
'end_ms': 0,
'start_ms': 0,
'type': 'stored'
}
}
self.assertDictEqual(d, query.request)
def test_and_filter(self):
andfilter = AndFilter([
InFilter(FieldValue(LogField.SRC), [IPValue("192.168.4.84")]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue("TCP/80")]),
])
f = {
"type":
"and",
"values": [
{
"left": {
"id": 7,
"type": "field"
},
"right": [{
"type": "ip",
"value": "192.168.4.84"
}],
"type": "in",
},
{
"left": {
"id": 27,
"type": "field"
},
"right": [{
"type": "service",
"value": "TCP/80"
}],
"type": "in",
},
],
}
self.assertDictEqual(andfilter.filter, f)
query = LogQuery()
query.add_and_filter([
InFilter(FieldValue(LogField.SRC), [IPValue("192.168.4.84")]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue("TCP/80")]),
])
query_filter = query.request["query"]["filter"]
self.assertDictEqual(f, query_filter)
# Update the original filter
andfilter.update_filter([
InFilter(FieldValue(LogField.DST), [IPValue("1.1.1.1")]),
InFilter(FieldValue(LogField.ACTION),
[ConstantValue(Actions.DISCARD, Actions.BLOCK)]),
])
d = {
"type":
"and",
"values": [
{
"left": {
"id": 8,
"type": "field"
},
"right": [{
"type": "ip",
"value": "1.1.1.1"
}],
"type": "in",
},
{
"left": {
"id": 14,
"type": "field"
},
"right": [{
"type": "constant",
"value": 0
}, {
"type": "constant",
"value": 13
}],
"type":
"in",
},
],
}
self.assertDictEqual(andfilter.filter, d)
query.update_filter(
AndFilter([
InFilter(FieldValue(LogField.DST), [IPValue("1.1.1.1")]),
InFilter(FieldValue(LogField.ACTION),
[ConstantValue(Actions.DISCARD, Actions.BLOCK)]),
]))
query_filter = query.request["query"]["filter"]
self.assertDictEqual(d, query_filter)
#query.request = {"query":{"definition":"BLACKLIST","target":"sg_vm"}, "fetch":{}, "format":{"type":"texts", "field_format": "name"}}
#query = UserQuery('lynn', check_hostname=False)
#query = SSLVPNQuery('lynn', check_hostname=False)
#query = RoutingQuery('lynn')
#print(Query.resolve_field_ids(BlacklistQuery.field_ids))
#query.request = {"query":{"definition":"BLACKLIST","target":"lynn"}, "fetch":{}, "format":{"type":"texts"}}
#myfilter = InFilter(FieldValue(LogField.SRC), [IPValue('192.168.4.82'), IPValue('172.18.1.152')])
#Use case to pull specific filter logs from Audit
query = LogQuery(backwards=False)
query.format.timezone('CET')
default_audit_fields_ids = [LogField.TIMESTAMP, LogField.DATATYPE, LogField.USERORIGINATOR, LogField.TYPEDESCRIPTION, LogField.RESULT, LogField.OBJECTNAME, LogField.OBJECTID, LogField.OBJECTTYPE, LogField.INFOMSG]
query.format.field_ids(default_audit_fields_ids)
# Show only Audit log entries
query.add_in_filter( FieldValue(LogField.DATATYPE), [ConstantValue(DataType.AUDIT)]) #HINT: it could be nice to have the Audit Data Type as constant in the LogField
for log in query.fetch_batch(CSVFormat):
print(log)
#OBJECTNAME
#for log in query.fetch_live():
# print(log)
#pprint(query._get_field_schema())
def test_custom_formats_for_query(self):
dt = DetailedFormat(timezone='CST',
time_show_zone=True,
ip_elements=True,
ip_locations=True)
# Add detailed format to a query
query = LogQuery(format=dt)
d = {'fetch': {'backwards': True},
'format': {'field_format': 'pretty',
'resolving': {'ip_elements': True,
'ip_locations': True,
'senders': True,
'time_show_zone': True,
'timezone': 'CST'},
'type': 'detailed'},
'query': {'end_ms': 0, 'start_ms': 0, 'type': 'stored'}}
self.assertDictEqual(d, query.request)
# Change the format for a query
rf = RawFormat(field_format='name')
rf.field_ids([LogField.SRC, LogField.DST])
query.update_format(rf)
request = {'fetch': {'backwards': True},
'format': {
'field_format': 'name',
'field_ids': [7, 8],
'type': 'raw'},
'query': {'end_ms': 0, 'start_ms': 0, 'type': 'stored'}}
self.assertDictEqual(request, query.request)
# Start with fresh Query and add a combined filter. A combined filter
# mixes individual formats together and when data is returned it's
# dict key will be the mapping to the format
text = TextFormat()
text.field_ids([LogField.TIMESTAMP])
detailed = DetailedFormat()
detailed.field_ids([LogField.SRC, LogField.DST])
combined = CombinedFormat(tformat=text, dformat=detailed)
query = LogQuery(fetch_size=1)
query.update_format(combined)
d = {'fetch': {'backwards': True, 'quantity': 1},
'format': {'formats': {'dformat': {'field_format': 'pretty',
'field_ids': [7, 8],
'resolving': {'senders': True},
'type': 'detailed'},
'tformat': {'field_format': 'pretty',
'field_ids': [1],
'resolving': {'senders': True},
'type': 'texts'}},
'type': 'combined'},
'query': {'end_ms': 0, 'start_ms': 0, 'type': 'stored'}}
self.assertDictEqual(d, query.request)
def test_custom_formats_for_query(self):
dt = DetailedFormat(timezone="CST",
time_show_zone=True,
ip_elements=True,
ip_locations=True)
# Add detailed format to a query
query = LogQuery(format=dt)
d = {
"fetch": {
"backwards": True
},
"format": {
"field_format": "pretty",
"resolving": {
"ip_elements": True,
"ip_locations": True,
"senders": True,
"time_show_zone": True,
"timezone": "CST",
},
"type": "detailed",
},
"query": {
"end_ms": 0,
"start_ms": 0,
"type": "stored"
},
}
self.assertDictEqual(d, query.request)
# Change the format for a query
rf = RawFormat(field_format="name")
rf.field_ids([LogField.SRC, LogField.DST])
query.update_format(rf)
request = {
"fetch": {
"backwards": True
},
"format": {
"field_format": "name",
"field_ids": [7, 8],
"type": "raw"
},
"query": {
"end_ms": 0,
"start_ms": 0,
"type": "stored"
},
}
self.assertDictEqual(request, query.request)
# Start with fresh Query and add a combined filter. A combined filter
# mixes individual formats together and when data is returned it's
# dict key will be the mapping to the format
text = TextFormat()
text.field_ids([LogField.TIMESTAMP])
detailed = DetailedFormat()
detailed.field_ids([LogField.SRC, LogField.DST])
combined = CombinedFormat(tformat=text, dformat=detailed)
query = LogQuery(fetch_size=1)
query.update_format(combined)
d = {
"fetch": {
"backwards": True,
"quantity": 1
},
"format": {
"formats": {
"dformat": {
"field_format": "pretty",
"field_ids": [7, 8],
"resolving": {
"senders": True
},
"type": "detailed",
},
"tformat": {
"field_format": "pretty",
"field_ids": [1],
"resolving": {
"senders": True
},
"type": "texts",
},
},
"type": "combined",
},
"query": {
"end_ms": 0,
"start_ms": 0,
"type": "stored"
},
}
self.assertDictEqual(d, query.request)
def test_and_filter(self):
andfilter = AndFilter([
InFilter(FieldValue(LogField.SRC), [IPValue('192.168.4.84')]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue('TCP/80')])
])
f = {
'type':
'and',
'values': [{
'left': {
'id': 7,
'type': 'field'
},
'right': [{
'type': 'ip',
'value': '192.168.4.84'
}],
'type': 'in'
}, {
'left': {
'id': 27,
'type': 'field'
},
'right': [{
'type': 'service',
'value': 'TCP/80'
}],
'type': 'in'
}]
}
self.assertDictEqual(andfilter.filter, f)
query = LogQuery()
query.add_and_filter([
InFilter(FieldValue(LogField.SRC), [IPValue('192.168.4.84')]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue('TCP/80')])
])
query_filter = query.request['query']['filter']
self.assertDictEqual(f, query_filter)
# Update the original filter
andfilter.update_filter([
InFilter(FieldValue(LogField.DST), [IPValue('1.1.1.1')]),
InFilter(FieldValue(LogField.ACTION),
[ConstantValue(Actions.DISCARD, Actions.BLOCK)])
])
d = {
'type':
'and',
'values': [{
'left': {
'id': 8,
'type': 'field'
},
'right': [{
'type': 'ip',
'value': '1.1.1.1'
}],
'type': 'in'
}, {
'left': {
'id': 14,
'type': 'field'
},
'right': [{
'type': 'constant',
'value': 0
}, {
'type': 'constant',
'value': 13
}],
'type':
'in'
}]
}
self.assertDictEqual(andfilter.filter, d)
query.update_filter(
AndFilter([
InFilter(FieldValue(LogField.DST), [IPValue('1.1.1.1')]),
InFilter(FieldValue(LogField.ACTION),
[ConstantValue(Actions.DISCARD, Actions.BLOCK)])
]))
query_filter = query.request['query']['filter']
self.assertDictEqual(d, query_filter)
def test_not_filter(self):
notfilter = NotFilter([
InFilter(FieldValue(LogField.SERVICE),
[ServiceValue('UDP/53', 'TCP/80')])
])
f = {
'type': 'not',
'value': {
'left': {
'id': 27,
'type': 'field'
},
'right': [{
'type': 'service',
'value': 'UDP/53'
}, {
'type': 'service',
'value': 'TCP/80'
}],
'type':
'in'
}
}
self.assertDictEqual(notfilter.filter, f)
query = LogQuery()
query.add_not_filter([
InFilter(FieldValue(LogField.SERVICE),
[ServiceValue('UDP/53', 'TCP/80')])
])
query_filter = query.request['query']['filter']
self.assertDictEqual(f, query_filter)
notfilter.update_filter([
InFilter(FieldValue(LogField.HTTPREQUESTHOST),
[StringValue('play.googleapis.com')])
])
d = {
'type': 'not',
'value': {
'left': {
'id': 1586,
'type': 'field'
},
'right': [{
'type': 'string',
'value': 'play.googleapis.com'
}],
'type': 'in'
}
}
self.assertDictEqual(d, notfilter.filter)
query.update_filter(
NotFilter([
InFilter(FieldValue(LogField.HTTPREQUESTHOST),
[StringValue('play.googleapis.com')])
]))
query_filter = query.request['query']['filter']
self.assertDictEqual(d, query_filter)
def run_query_and_upload():
aws_integration = cfg.get('aws-integration', False)
azure_integration = cfg.get('azure-integration', False)
# Create default insights if they don't already exist
if aws_integration:
threading.Thread(target=create_default_insights).start()
setup_sec_hub()
enable_batch_import_findings()
default_filter = __setup_smc_query_filter(cfg['default-filter'])
smc_url = cfg['host-ip'] + ':' + cfg['host-port']
api_version = __get_latest_api_version(smc_url)
session.login(url=smc_url, api_key=cfg['client-api-key'], api_version=api_version)
try:
query = LogQuery(fetch_size=int(cfg['fetch-size']))
translated_filter = query.add_translated_filter()
# Create default filter specified in the config file
translated_filter.update_filter(default_filter)
# Query the SMC for events matching the filter and flatten the list of result-lists into a single list
record_list = list(itertools.chain(*query.fetch_raw()))
extra_filters_enabled = cfg['extra-filters-enabled']
# Check to see if extra filters are enabled and if any are present before iterating over them and requesting
# matching events from the SMC and appending to the original results list
if bool(extra_filters_enabled):
extra_filters = cfg['extra-filters']
if bool(extra_filters):
for log_filter in extra_filters:
translated_filter.update_filter(__setup_smc_query_filter(log_filter))
record_list.extend(list(itertools.chain(*query.fetch_raw())))
if record_list:
# Find the max date in the record list and store this to add to the filter for subsequent queries
# to avoid uploading duplicates/wasting bandwidth. This value is written to the cfg.json file
max_finding_date = format_date_smc_filter(max(item['Creation Time'] for item in record_list))
loop = asyncio.get_event_loop()
# Map to appropriate format and upload if integration is active
if aws_integration:
aws_task = loop.create_task(
amazon_security_hub_batch_upload(list(map(create_asff_object, record_list)), max_finding_date))
loop.run_until_complete(aws_task)
if azure_integration:
send_sentinel_data(list(map(format_smc_logs_to_cef, record_list)), max_finding_date)
# This catches any issues related to requesting events with a malformed filter
except FetchAborted as exception:
print(exception)
write_to_log(exception)
session.logout()
def test_not_filter(self):
notfilter = NotFilter([
InFilter(FieldValue(LogField.SERVICE),
[ServiceValue("UDP/53", "TCP/80")])
])
f = {
"type": "not",
"value": {
"left": {
"id": 27,
"type": "field"
},
"right": [
{
"type": "service",
"value": "UDP/53"
},
{
"type": "service",
"value": "TCP/80"
},
],
"type":
"in",
},
}
self.assertDictEqual(notfilter.filter, f)
query = LogQuery()
query.add_not_filter([
InFilter(FieldValue(LogField.SERVICE),
[ServiceValue("UDP/53", "TCP/80")])
])
query_filter = query.request["query"]["filter"]
self.assertDictEqual(f, query_filter)
notfilter.update_filter([
InFilter(FieldValue(LogField.HTTPREQUESTHOST),
[StringValue("play.googleapis.com")])
])
d = {
"type": "not",
"value": {
"left": {
"id": 1586,
"type": "field"
},
"right": [{
"type": "string",
"value": "play.googleapis.com"
}],
"type": "in",
},
}
self.assertDictEqual(d, notfilter.filter)
query.update_filter(
NotFilter([
InFilter(FieldValue(LogField.HTTPREQUESTHOST),
[StringValue("play.googleapis.com")])
]))
query_filter = query.request["query"]["filter"]
self.assertDictEqual(d, query_filter)
if __name__ == '__main__':
#session.login(url='http://172.18.1.26:8082', api_key='kKphtsbQKjjfHR7amodA0001', timeout=45,
# beta=True)
session.login(url='http://172.18.1.150:8082', api_key='EiGpKD4QxlLJ25dbBEp20001', timeout=30)
#session.login(url='https://172.18.1.151:8082', api_key='xJRo27kGja4JmPek9l3Nyxm4',
# verify=False)
pprint(session._get_log_schema())
#TODO: BLACKLISTQUERY fails when using format ID's due to CombinedFilter.
query = LogQuery(http_proxy_host='1.1.1.1')
for log in query.fetch_live():
print(log)
sys.exit(1)
websocket.enableTrace(True)
#os.kill(os.getpid(), signal.SIGTERM)
#print(os.getpid())
#subscribe_policy()
field_ids = ConnectionQuery.field_ids
field_ids.remove(LogField.TIMESTAMP)
field_ids.remove(LogField.NODEID)
field_ids.extend((LogField.NATSRC, LogField.NATDPORT))
if __name__ == '__main__':
#session.login(url='http://172.18.1.26:8082', api_key='kKphtsbQKjjfHR7amodA0001', timeout=45,
# beta=True)
session.login(url='http://172.18.1.150:8082',
api_key='EiGpKD4QxlLJ25dbBEp20001',
timeout=30)
#session.login(url='https://172.18.1.151:8082', api_key='xJRo27kGja4JmPek9l3Nyxm4',
# verify=False)
pprint(session._get_log_schema())
#TODO: BLACKLISTQUERY fails when using format ID's due to CombinedFilter.
query = LogQuery(http_proxy_host='1.1.1.1')
for log in query.fetch_live():
print(log)
sys.exit(1)
websocket.enableTrace(True)
#os.kill(os.getpid(), signal.SIGTERM)
#print(os.getpid())
#subscribe_policy()
field_ids = ConnectionQuery.field_ids
field_ids.remove(LogField.TIMESTAMP)
field_ids.remove(LogField.NODEID)
field_ids.extend((LogField.NATSRC, LogField.NATDPORT))
def test_or_filter(self):
orfilter = OrFilter([
InFilter(FieldValue(LogField.SRC), [IPValue("192.168.4.84")]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue("TCP/80")]),
])
f = {
"type":
"or",
"values": [
{
"left": {
"id": 7,
"type": "field"
},
"right": [{
"type": "ip",
"value": "192.168.4.84"
}],
"type": "in",
},
{
"left": {
"id": 27,
"type": "field"
},
"right": [{
"type": "service",
"value": "TCP/80"
}],
"type": "in",
},
],
}
self.assertDictEqual(orfilter.filter, f)
query = LogQuery()
query.add_or_filter([
InFilter(FieldValue(LogField.SRC), [IPValue("192.168.4.84")]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue("TCP/80")]),
])
query_filter = query.request["query"]["filter"]
self.assertDictEqual(f, query_filter)
orfilter.update_filter([
InFilter(FieldValue(LogField.DST), [IPValue("1.1.1.1")]),
InFilter(FieldValue(LogField.SERVICE), [ServiceValue("TCP/443")]),
])
d = {
"type":
"or",
"values": [
{
"left": {
"id": 8,
"type": "field"
},
"right": [{
"type": "ip",
"value": "1.1.1.1"
}],
"type": "in",
},
{
"left": {
"id": 27,
"type": "field"
},
"right": [{
"type": "service",
"value": "TCP/443"
}],
"type": "in",
},
],
}
self.assertDictEqual(d, orfilter.filter)
query.update_filter(
OrFilter([
InFilter(FieldValue(LogField.DST), [IPValue("1.1.1.1")]),
InFilter(FieldValue(LogField.SERVICE),
[ServiceValue("TCP/443")]),
]))
query_filter = query.request["query"]["filter"]
self.assertDictEqual(d, query_filter)