def test_filter_list(self):
parser = flat_file.FlatFile('./tests/data/cookies.txt')
parser.add_filters({'ip': ['X.X.X.X', 'A.A.A.A']})
self.assertEquals([i for i in parser.get_results()],
[{
'block': '0',
'cscore0': '$UWA',
'id': '42000227',
'ip': 'X.X.X.X',
'learning': '0',
'score0': '8',
'server': 'Y.Y.Y.Y',
'total_blocked': '204',
'total_processed': '472',
'uri': '/phpMyAdmin-2.8.2/scripts/setup.php',
'var_name': 'cookie',
'vers': '0.52',
'zone': 'HEADERS',
'coords': None,
'date': '20131110T07:36:19'
}])
parser = flat_file.FlatFile('./tests/data/cookies.txt')
parser.add_filters({'ip': ['A.A.A.A']})
self.assertEquals([i for i in parser.get_results()], [])
parser = flat_file.FlatFile('./tests/data/cookies.txt')
parser.add_filters({'ip': ['X.X.X.X']}, negative=True)
self.assertEquals([i for i in parser.get_results()], [])
def main():
args = __create_argparser()
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
logging.getLogger("urllib3").setLevel(logging.ERROR)
if args.verbose:
logging.basicConfig(level=logging.DEBUG, format='%(message)s')
else:
logging.basicConfig(level=logging.INFO, format='[+] %(message)s')
if args.elastic is True:
if elastic_imported is False:
print(
'You asked for an elastic source, but you do not have the required dependencies.'
)
return
source = elastic.Elastic()
elif args.flat_file:
source = flat_file.FlatFile(args.flat_file)
elif args.stdin is True:
source = flat_file.FlatFile()
else:
print(
'Please give me a valid source (or try to relaunch me with `-h` if you are lost).'
)
return 1
# Filtering can be used for any operation
__filter(source, args.filter, regexp=False, hostname=args.hostname)
if args.filter_regexp:
__filter(source,
args.filter_regexp,
regexp=True,
hostname=args.hostname)
if args.stats:
printers.print_statistics(source.get_statistics())
elif args.whitelist:
whitelist = list()
for module in WL_MODULES:
rules = module.generate_whitelist(source, whitelist)
whitelist.extend(rules)
__whitelist_from_rules(source, rules)
if whitelist:
print('\n\033[1mGenerated whitelists:\033[0m')
print('\t' +
';\n\t'.join(map(nxapi_whitelist.dict_to_str, whitelist)) +
';')
else:
print(
'\n\033[1mnxtool was not able to generate meaningful whitelist\033[0m'
)
elif args.typing:
printers.print_typed_rules(typing.typification(source))
else:
print(printers.print_generic(source.get_results()))
def test_generate_whitelist_zone_var_wide_url(self):
parser = flat_file.FlatFile('./tests/data/images_1002.txt')
parser.get_relevant_ids = lambda x: [1337]
parser.get_top = lambda x: {'1337': 2048, '': 123, 'test': 1}
expected = [{
'msg':
'Variable zone-wide on a specific url if it matches a id 1337',
'mz': ['$URL:1337|$BODY_VAR:1337'],
'wl': [1337]
}, {
'msg':
'Variable zone-wide on a specific url if it matches a id 1337',
'mz': ['$URL:1337|ARGS:1337|NAME'],
'wl': [1337]
}, {
'msg':
'Variable zone-wide on a specific url if it matches a id 1337',
'mz': ['$URL:1337|$ARGS_VAR:1337'],
'wl': [1337]
}, {
'msg':
'Variable zone-wide on a specific url if it matches a id 1337',
'mz': ['$URL:1337|BODY:1337|NAME'],
'wl': [1337]
}]
self.assertTrue(
all(i in zone_var_wide_url.generate_whitelist(
parser, [{
'id': 123
}]) for i in expected))
def test_generate_whitelist_google_analytics(self):
parser = flat_file.FlatFile('./tests/data/images_1002.txt')
parser.get_relevant_ids = lambda x: [1337]
parser.get_top = lambda x: {'1337': 2048}
self.assertEqual(google_analytics.generate_whitelist(parser, []),
[{
'msg': 'Google analytics',
'mz': ['$ARGS_VAR_X:__utm[abctvz]'],
'wl': [1337]
}])
self.assertEqual(
google_analytics.generate_whitelist(parser, [{
'id': 1234
}]), [{
'msg': 'Google analytics',
'mz': ['$ARGS_VAR_X:__utm[abctvz]'],
'wl': [1337]
}])
self.assertEqual(
google_analytics.generate_whitelist(parser, [{
'wl': 1002
}]), [{
'msg': 'Google analytics',
'mz': ['$ARGS_VAR_X:__utm[abctvz]'],
'wl': [1337]
}])
def test_get_relevant_ids(self):
parser = flat_file.FlatFile('./tests/data/cookies.txt')
self.assertEquals(parser.get_relevant_ids([
'zone',
], 1, 1), [42000227])
self.assertEquals(parser.get_relevant_ids([
'zone',
], 1, 100), [])
def test_typing(self):
parser = flat_file.FlatFile('./tests/data/exlog.txt')
self.assertEquals([i for i in typing.typification(parser)],
[['^\\d+$', 'integer', 'ARGS', 'a']])
parser.get_results = lambda: [
{
'zone': "BODY",
'var_name': "pif"
},
]
self.assertFalse([i for i in typing.typification(parser)])
def test_generate_whitelist_images(self):
parser = flat_file.FlatFile('./tests/data/images_1002.txt')
self.assertEqual(
images_1002.generate_whitelist(parser, []),
[{
'mz': ['$URL_X:^/phpMyAdmin-2.8.2/scripts/setup.php|URL'],
'wl': [1002],
'msg': 'Images size (0x)'
}])
self.assertEqual(
images_1002.generate_whitelist(parser, [{
'wl': [1002]
}]), [])
def test_generate_whitelist_cookies(self):
parser = flat_file.FlatFile('./tests/data/cookies.txt')
parser.get_relevant_ids = lambda x: [42000227]
self.assertEqual(cookies.generate_whitelist(parser, []),
[{
'wl': [42000227],
'mz': ['$HEADERS_VAR:cookie'],
'msg': 'Cookies that matches a id 42000227'
}])
self.assertEqual(cookies.generate_whitelist(parser, [{
'id': 1234
}]), [{
'wl': [42000227],
'mz': ['$HEADERS_VAR:cookie'],
'msg': 'Cookies that matches a id 42000227'
}])
def test_elastic_import(self):
dest = elastic.Elastic()
source = flat_file.FlatFile('./tests/data/exlog.txt')
for log in source.logs:
dest.insert([log])
dest.stop()
dest.initialize_search()
dest.minimum_occurences = 0
dest.percentage = 0
time.sleep(5)
self.assertEqual(dest.get_relevant_ids(['id']), {u'1302', u'42000227'})
self.assertEqual(dest.get_top('id'), {1302: 3, 42000227: 1})
self.assertEqual(dest.get_top('uri'), {
u'/': 3,
u'/phpMyAdmin-2.8.2/scripts/setup.php': 1
})
dest.client.indices.delete(index=dest.index, ignore=[400, 404])
def test_generate_whitelist_zone_var_wide(self):
parser = flat_file.FlatFile('./tests/data/images_1002.txt')
parser.get_relevant_ids = lambda x: [1337]
parser.get_top = lambda x: {
'test_var_name': 2048,
'': 123,
'super-test': 12
}
expected = [{
'msg': 'Variable zone-wide if it matches a id 1337',
'mz': ['BODY:test_var_name'],
'wl': [1337]
}, {
'msg': 'Variable zone-wide if it matches a id 1337',
'mz': ['ARGS:test_var_name|NAME'],
'wl': [1337]
}, {
'msg': 'Variable zone-wide if it matches a id 1337',
'mz': ['ARGS:test_var_name'],
'wl': [1337]
}, {
'msg': 'Variable zone-wide if it matches a id 1337',
'mz': ['BODY:test_var_name|NAME'],
'wl': [1337]
}]
try:
self.assertCountEqual(zone_var_wide.generate_whitelist(parser, []),
expected)
except AttributeError: # Python2/3 fuckery
self.assertItemsEqual(zone_var_wide.generate_whitelist(parser, []),
expected)
parser.get_top = lambda x: {'test_var_name': 0}
try:
self.assertCountEqual(
zone_var_wide.generate_whitelist(parser, [{
'id': 1,
'mz': 'BODY'
}]), [])
except AttributeError: # Python2/3 fuckery
self.assertItemsEqual(
zone_var_wide.generate_whitelist(parser, [{
'id': 1,
'mz': 'BODY'
}]), [])
def test_generate_whitelist_array_like_variables_names(self):
parser = flat_file.FlatFile('./tests/data/images_1002.txt')
parser.get_relevant_ids = lambda x: [1337]
parser.get_top = lambda x: {'1337': 2048}
self.assertEqual(
array_like_variables_names.generate_whitelist(parser, []), [])
parser.get_relevant_ids = lambda x: [1310]
parser.get_top = lambda x: {'test[1234]': 2048}
self.assertEqual(
array_like_variables_names.generate_whitelist(parser, []),
[{
'msg': 'Array-like variable name',
'mz': ['$BODY_VAR_X:^test\\[.+\\]$'],
'wl': [1310, 1311]
}, {
'msg': 'Array-like variable name',
'mz': ['$ARGS_VAR_X:^test\\[.+\\]$'],
'wl': [1310, 1311]
}])
self.assertEqual(
array_like_variables_names.generate_whitelist(
parser, [{
'id': 3,
'mz': 'BODY'
}]), [{
'msg': 'Array-like variable name',
'mz': ['$BODY_VAR_X:^test\\[.+\\]$'],
'wl': [1310, 1311]
}, {
'msg': 'Array-like variable name',
'mz': ['$ARGS_VAR_X:^test\\[.+\\]$'],
'wl': [1310, 1311]
}])
parser.get_top = lambda x: {'test[1234]]': 2048}
self.assertEqual(
array_like_variables_names.generate_whitelist(parser, []), [])
parser.get_top = lambda x: {'test[[1234]': 2048}
self.assertEqual(
array_like_variables_names.generate_whitelist(parser, []), [])
parser.get_top = lambda x: {'test[1234]': 1}
self.assertEqual(
array_like_variables_names.generate_whitelist(parser, []), [])
def test_generate_whitelist_site_wide_id(self):
parser = flat_file.FlatFile('./tests/data/images_1002.txt')
parser.get_relevant_ids = lambda x, y: [1337]
parser.get_top = lambda x: {'1337': 2048}
self.assertEqual(site_wide_id.generate_whitelist(parser, []),
[{
'msg': 'Site-wide id+zone if it matches id 1337',
'mz': ['1337'],
'wl': [1337]
}])
self.assertEqual(
site_wide_id.generate_whitelist(parser, [{
'id': 1234
}]), [{
'msg': 'Site-wide id+zone if it matches id 1337',
'mz': ['1337'],
'wl': [1337]
}])
def test_filter_str(self):
parser = flat_file.FlatFile('./tests/data/cookies.txt')
self.assertEquals(
[i for i in parser.get_results()][0], {
'block': '0',
'cscore0': '$UWA',
'id0': '42000227',
'ip': 'X.X.X.X',
'learning': '0',
'score0': '8',
'server': 'Y.Y.Y.Y',
'total_blocked': '204',
'total_processed': '472',
'uri': '/phpMyAdmin-2.8.2/scripts/setup.php',
'var_name0': 'cookie',
'vers': '0.52',
'zone0': 'HEADERS'
})
def test_generate_whitelist_url_wide(self):
parser = flat_file.FlatFile('./tests/data/images_1002.txt')
parser.get_relevant_ids = lambda x: [1337]
parser.get_top = lambda x: {'1337': 2048, '123': 2}
expected = [{
'msg': 'url-wide whitelist if it matches a id 1337',
'mz': ['$URL:1337'],
'wl': {'1337'}
}, {
'msg': 'url-wide whitelist if it matches a id 1337',
'mz': ['$URL:123'],
'wl': {'1337'}
}]
self.assertTrue(
all(i in url_wide.generate_whitelist(parser, [])
for i in expected))
parser.get_relevant_ids = lambda x: []
parser.get_top = lambda x: {}
self.assertEqual(url_wide.generate_whitelist(parser, [{
'id': 1337
}]), [])
def test_generate_whitelist_zone_wide(self):
parser = flat_file.FlatFile('./tests/data/images_1002.txt')
parser.get_top = lambda x: {
1337: 2048,
'': 1337
} if x == 'id' else {
'ARGS': 2048
}
parser.get_relevant_ids = lambda x: [1337]
self.assertEqual(zone_wide.generate_whitelist(
parser, []), [{
'msg': 'zone-wide ID whitelist if it matches a id 1337',
'mz': ['ARGS'],
'wl': {1337}
}])
parser.get_relevant_ids = lambda x: []
self.assertEqual(zone_wide.generate_whitelist(parser, [{
'id': 1337
}]), [])
parser.get_top = lambda x: {1337: 2} if x == 'id' else {'ARGS': 2}
self.assertEqual(zone_wide.generate_whitelist(parser, []), [])
def test_show_stats(self):
parser = flat_file.FlatFile('tests/data/logs.txt')
parser.get_statistics()
