def test_filter_list(self): parser = flat_file.FlatFile('./tests/data/cookies.txt') parser.add_filters({'ip': ['X.X.X.X', 'A.A.A.A']}) self.assertEquals([i for i in parser.get_results()], [{ 'block': '0', 'cscore0': '$UWA', 'id': '42000227', 'ip': 'X.X.X.X', 'learning': '0', 'score0': '8', 'server': 'Y.Y.Y.Y', 'total_blocked': '204', 'total_processed': '472', 'uri': '/phpMyAdmin-2.8.2/scripts/setup.php', 'var_name': 'cookie', 'vers': '0.52', 'zone': 'HEADERS', 'coords': None, 'date': '20131110T07:36:19' }]) parser = flat_file.FlatFile('./tests/data/cookies.txt') parser.add_filters({'ip': ['A.A.A.A']}) self.assertEquals([i for i in parser.get_results()], []) parser = flat_file.FlatFile('./tests/data/cookies.txt') parser.add_filters({'ip': ['X.X.X.X']}, negative=True) self.assertEquals([i for i in parser.get_results()], [])
def main(): args = __create_argparser() logging.getLogger("elasticsearch").setLevel(logging.ERROR) logging.getLogger("urllib3").setLevel(logging.ERROR) if args.verbose: logging.basicConfig(level=logging.DEBUG, format='%(message)s') else: logging.basicConfig(level=logging.INFO, format='[+] %(message)s') if args.elastic is True: if elastic_imported is False: print( 'You asked for an elastic source, but you do not have the required dependencies.' ) return source = elastic.Elastic() elif args.flat_file: source = flat_file.FlatFile(args.flat_file) elif args.stdin is True: source = flat_file.FlatFile() else: print( 'Please give me a valid source (or try to relaunch me with `-h` if you are lost).' ) return 1 # Filtering can be used for any operation __filter(source, args.filter, regexp=False, hostname=args.hostname) if args.filter_regexp: __filter(source, args.filter_regexp, regexp=True, hostname=args.hostname) if args.stats: printers.print_statistics(source.get_statistics()) elif args.whitelist: whitelist = list() for module in WL_MODULES: rules = module.generate_whitelist(source, whitelist) whitelist.extend(rules) __whitelist_from_rules(source, rules) if whitelist: print('\n\033[1mGenerated whitelists:\033[0m') print('\t' + ';\n\t'.join(map(nxapi_whitelist.dict_to_str, whitelist)) + ';') else: print( '\n\033[1mnxtool was not able to generate meaningful whitelist\033[0m' ) elif args.typing: printers.print_typed_rules(typing.typification(source)) else: print(printers.print_generic(source.get_results()))
def test_generate_whitelist_zone_var_wide_url(self): parser = flat_file.FlatFile('./tests/data/images_1002.txt') parser.get_relevant_ids = lambda x: [1337] parser.get_top = lambda x: {'1337': 2048, '': 123, 'test': 1} expected = [{ 'msg': 'Variable zone-wide on a specific url if it matches a id 1337', 'mz': ['$URL:1337|$BODY_VAR:1337'], 'wl': [1337] }, { 'msg': 'Variable zone-wide on a specific url if it matches a id 1337', 'mz': ['$URL:1337|ARGS:1337|NAME'], 'wl': [1337] }, { 'msg': 'Variable zone-wide on a specific url if it matches a id 1337', 'mz': ['$URL:1337|$ARGS_VAR:1337'], 'wl': [1337] }, { 'msg': 'Variable zone-wide on a specific url if it matches a id 1337', 'mz': ['$URL:1337|BODY:1337|NAME'], 'wl': [1337] }] self.assertTrue( all(i in zone_var_wide_url.generate_whitelist( parser, [{ 'id': 123 }]) for i in expected))
def test_generate_whitelist_google_analytics(self): parser = flat_file.FlatFile('./tests/data/images_1002.txt') parser.get_relevant_ids = lambda x: [1337] parser.get_top = lambda x: {'1337': 2048} self.assertEqual(google_analytics.generate_whitelist(parser, []), [{ 'msg': 'Google analytics', 'mz': ['$ARGS_VAR_X:__utm[abctvz]'], 'wl': [1337] }]) self.assertEqual( google_analytics.generate_whitelist(parser, [{ 'id': 1234 }]), [{ 'msg': 'Google analytics', 'mz': ['$ARGS_VAR_X:__utm[abctvz]'], 'wl': [1337] }]) self.assertEqual( google_analytics.generate_whitelist(parser, [{ 'wl': 1002 }]), [{ 'msg': 'Google analytics', 'mz': ['$ARGS_VAR_X:__utm[abctvz]'], 'wl': [1337] }])
def test_get_relevant_ids(self): parser = flat_file.FlatFile('./tests/data/cookies.txt') self.assertEquals(parser.get_relevant_ids([ 'zone', ], 1, 1), [42000227]) self.assertEquals(parser.get_relevant_ids([ 'zone', ], 1, 100), [])
def test_typing(self): parser = flat_file.FlatFile('./tests/data/exlog.txt') self.assertEquals([i for i in typing.typification(parser)], [['^\\d+$', 'integer', 'ARGS', 'a']]) parser.get_results = lambda: [ { 'zone': "BODY", 'var_name': "pif" }, ] self.assertFalse([i for i in typing.typification(parser)])
def test_generate_whitelist_images(self): parser = flat_file.FlatFile('./tests/data/images_1002.txt') self.assertEqual( images_1002.generate_whitelist(parser, []), [{ 'mz': ['$URL_X:^/phpMyAdmin-2.8.2/scripts/setup.php|URL'], 'wl': [1002], 'msg': 'Images size (0x)' }]) self.assertEqual( images_1002.generate_whitelist(parser, [{ 'wl': [1002] }]), [])
def test_generate_whitelist_cookies(self): parser = flat_file.FlatFile('./tests/data/cookies.txt') parser.get_relevant_ids = lambda x: [42000227] self.assertEqual(cookies.generate_whitelist(parser, []), [{ 'wl': [42000227], 'mz': ['$HEADERS_VAR:cookie'], 'msg': 'Cookies that matches a id 42000227' }]) self.assertEqual(cookies.generate_whitelist(parser, [{ 'id': 1234 }]), [{ 'wl': [42000227], 'mz': ['$HEADERS_VAR:cookie'], 'msg': 'Cookies that matches a id 42000227' }])
def test_elastic_import(self): dest = elastic.Elastic() source = flat_file.FlatFile('./tests/data/exlog.txt') for log in source.logs: dest.insert([log]) dest.stop() dest.initialize_search() dest.minimum_occurences = 0 dest.percentage = 0 time.sleep(5) self.assertEqual(dest.get_relevant_ids(['id']), {u'1302', u'42000227'}) self.assertEqual(dest.get_top('id'), {1302: 3, 42000227: 1}) self.assertEqual(dest.get_top('uri'), { u'/': 3, u'/phpMyAdmin-2.8.2/scripts/setup.php': 1 }) dest.client.indices.delete(index=dest.index, ignore=[400, 404])
def test_generate_whitelist_zone_var_wide(self): parser = flat_file.FlatFile('./tests/data/images_1002.txt') parser.get_relevant_ids = lambda x: [1337] parser.get_top = lambda x: { 'test_var_name': 2048, '': 123, 'super-test': 12 } expected = [{ 'msg': 'Variable zone-wide if it matches a id 1337', 'mz': ['BODY:test_var_name'], 'wl': [1337] }, { 'msg': 'Variable zone-wide if it matches a id 1337', 'mz': ['ARGS:test_var_name|NAME'], 'wl': [1337] }, { 'msg': 'Variable zone-wide if it matches a id 1337', 'mz': ['ARGS:test_var_name'], 'wl': [1337] }, { 'msg': 'Variable zone-wide if it matches a id 1337', 'mz': ['BODY:test_var_name|NAME'], 'wl': [1337] }] try: self.assertCountEqual(zone_var_wide.generate_whitelist(parser, []), expected) except AttributeError: # Python2/3 fuckery self.assertItemsEqual(zone_var_wide.generate_whitelist(parser, []), expected) parser.get_top = lambda x: {'test_var_name': 0} try: self.assertCountEqual( zone_var_wide.generate_whitelist(parser, [{ 'id': 1, 'mz': 'BODY' }]), []) except AttributeError: # Python2/3 fuckery self.assertItemsEqual( zone_var_wide.generate_whitelist(parser, [{ 'id': 1, 'mz': 'BODY' }]), [])
def test_generate_whitelist_array_like_variables_names(self): parser = flat_file.FlatFile('./tests/data/images_1002.txt') parser.get_relevant_ids = lambda x: [1337] parser.get_top = lambda x: {'1337': 2048} self.assertEqual( array_like_variables_names.generate_whitelist(parser, []), []) parser.get_relevant_ids = lambda x: [1310] parser.get_top = lambda x: {'test[1234]': 2048} self.assertEqual( array_like_variables_names.generate_whitelist(parser, []), [{ 'msg': 'Array-like variable name', 'mz': ['$BODY_VAR_X:^test\\[.+\\]$'], 'wl': [1310, 1311] }, { 'msg': 'Array-like variable name', 'mz': ['$ARGS_VAR_X:^test\\[.+\\]$'], 'wl': [1310, 1311] }]) self.assertEqual( array_like_variables_names.generate_whitelist( parser, [{ 'id': 3, 'mz': 'BODY' }]), [{ 'msg': 'Array-like variable name', 'mz': ['$BODY_VAR_X:^test\\[.+\\]$'], 'wl': [1310, 1311] }, { 'msg': 'Array-like variable name', 'mz': ['$ARGS_VAR_X:^test\\[.+\\]$'], 'wl': [1310, 1311] }]) parser.get_top = lambda x: {'test[1234]]': 2048} self.assertEqual( array_like_variables_names.generate_whitelist(parser, []), []) parser.get_top = lambda x: {'test[[1234]': 2048} self.assertEqual( array_like_variables_names.generate_whitelist(parser, []), []) parser.get_top = lambda x: {'test[1234]': 1} self.assertEqual( array_like_variables_names.generate_whitelist(parser, []), [])
def test_generate_whitelist_site_wide_id(self): parser = flat_file.FlatFile('./tests/data/images_1002.txt') parser.get_relevant_ids = lambda x, y: [1337] parser.get_top = lambda x: {'1337': 2048} self.assertEqual(site_wide_id.generate_whitelist(parser, []), [{ 'msg': 'Site-wide id+zone if it matches id 1337', 'mz': ['1337'], 'wl': [1337] }]) self.assertEqual( site_wide_id.generate_whitelist(parser, [{ 'id': 1234 }]), [{ 'msg': 'Site-wide id+zone if it matches id 1337', 'mz': ['1337'], 'wl': [1337] }])
def test_filter_str(self): parser = flat_file.FlatFile('./tests/data/cookies.txt') self.assertEquals( [i for i in parser.get_results()][0], { 'block': '0', 'cscore0': '$UWA', 'id0': '42000227', 'ip': 'X.X.X.X', 'learning': '0', 'score0': '8', 'server': 'Y.Y.Y.Y', 'total_blocked': '204', 'total_processed': '472', 'uri': '/phpMyAdmin-2.8.2/scripts/setup.php', 'var_name0': 'cookie', 'vers': '0.52', 'zone0': 'HEADERS' })
def test_generate_whitelist_url_wide(self): parser = flat_file.FlatFile('./tests/data/images_1002.txt') parser.get_relevant_ids = lambda x: [1337] parser.get_top = lambda x: {'1337': 2048, '123': 2} expected = [{ 'msg': 'url-wide whitelist if it matches a id 1337', 'mz': ['$URL:1337'], 'wl': {'1337'} }, { 'msg': 'url-wide whitelist if it matches a id 1337', 'mz': ['$URL:123'], 'wl': {'1337'} }] self.assertTrue( all(i in url_wide.generate_whitelist(parser, []) for i in expected)) parser.get_relevant_ids = lambda x: [] parser.get_top = lambda x: {} self.assertEqual(url_wide.generate_whitelist(parser, [{ 'id': 1337 }]), [])
def test_generate_whitelist_zone_wide(self): parser = flat_file.FlatFile('./tests/data/images_1002.txt') parser.get_top = lambda x: { 1337: 2048, '': 1337 } if x == 'id' else { 'ARGS': 2048 } parser.get_relevant_ids = lambda x: [1337] self.assertEqual(zone_wide.generate_whitelist( parser, []), [{ 'msg': 'zone-wide ID whitelist if it matches a id 1337', 'mz': ['ARGS'], 'wl': {1337} }]) parser.get_relevant_ids = lambda x: [] self.assertEqual(zone_wide.generate_whitelist(parser, [{ 'id': 1337 }]), []) parser.get_top = lambda x: {1337: 2} if x == 'id' else {'ARGS': 2} self.assertEqual(zone_wide.generate_whitelist(parser, []), [])
def test_show_stats(self): parser = flat_file.FlatFile('tests/data/logs.txt') parser.get_statistics()