class TestSessionAnalyzer(unittest.TestCase):
def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.session = json.loads(session.decode('utf-8'))
self.handler = SessionAnalyzer(loop=self.loop)
def tests_load_session_fail(self):
async def sess_get(key):
return asyncio_redis.NotConnectedError
redis_mock = mock.Mock()
redis_mock.get = sess_get
res = None
with self.assertLogs():
self.loop.run_until_complete(self.handler.analyze(
None, redis_mock))
def test_create_stats(self):
async def sess_get():
return session
async def set_of_members(key):
return set()
async def push_list():
return ''
redis_mock = mock.Mock()
redis_mock.get = sess_get
redis_mock.smembers_asset = set_of_members
redis_mock.lpush = push_list
stats = self.loop.run_until_complete(
self.handler.create_stats(self.session, redis_mock))
self.assertEqual(stats['possible_owners'], ['attacker'])
class TestSessionAnalyzer(unittest.TestCase):
def setUp(self):
self.session = json.loads(session.decode('utf-8'))
self.handler = SessionAnalyzer()
def tests_load_session_fail(self):
@asyncio.coroutine
def sess_get():
return asyncio_redis.NotConnectedError
redis_mock = mock.Mock()
redis_mock.get = sess_get
res = None
loop = asyncio.get_event_loop()
redis_mock = mock.Mock()
redis_mock.get = sess_get
loop.run_until_complete(self.handler.analyze(None, redis_mock))
self.assertRaises(asyncio_redis.NotConnectedError)
def test_create_stats(self):
@asyncio.coroutine
def sess_get():
return session
@asyncio.coroutine
def set_of_members(key):
return set()
@asyncio.coroutine
def push_list():
return ''
redis_mock = mock.Mock()
redis_mock.get = sess_get
redis_mock.smembers_asset = set_of_members
redis_mock.lpush = push_list
stats = asyncio.get_event_loop().run_until_complete(self.handler.create_stats(self.session, redis_mock))
self.assertEqual(stats['possible_owners'], ['attacker'])
def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.session = json.loads(session.decode('utf-8'))
self.handler = SessionAnalyzer(loop=self.loop)
self.res = None
geoip2.database.Reader.__init__ = Mock(return_value=None)
rvalue = geoip2.models.City(
{'city': {'geoname_id': 4223379, 'names': {'en': 'Smyrna',
'ru': 'Смирна', 'zh-CN': '士麦那'}},
'continent': {'code': 'NA', 'geoname_id': 6255149,
'names': {'de': 'Nordamerika', 'en': 'North America',
'es': 'Norteamérica', 'fr': 'Amérique du Nord',
'ja': '北アメリカ', 'pt-BR': 'América do Norte',
'ru': 'Северная Америка', 'zh-CN': '北美洲'}},
'country': {'geoname_id': 6252001, 'iso_code': 'US',
'names': {'de': 'USA', 'en': 'United States',
'es': 'Estados Unidos', 'fr': 'États-Unis',
'ja': 'アメリカ合衆国', 'pt-BR': 'Estados Unidos',
'ru': 'США', 'zh-CN': '美国'}},
'location': {'accuracy_radius': 10,
'latitude': 33.8633,
'longitude': -84.4984,
'metro_code': 524,
'time_zone': 'America/New_York'},
'postal': {'code': '30080'},
'registered_country': {'geoname_id': 6252001, 'iso_code': 'US',
'names': {'de': 'USA', 'en': 'United States',
'es': 'Estados Unidos', 'fr': 'États-Unis',
'ja': 'アメリカ合衆国', 'pt-BR': 'Estados Unidos',
'ru': 'США', 'zh-CN': '美国'}},
'subdivisions': [{'geoname_id': 4197000, 'iso_code': 'GA',
'names': {'en': 'Georgia', 'es': 'Georgia',
'fr': 'Géorgie', 'ja': 'ジョージア州',
'pt-BR': 'Geórgia', 'ru': 'Джорджия',
'zh-CN': '乔治亚'}}],
'traits': {'ip_address': '74.217.37.8'}}, ['en']
)
geoip2.database.Reader.city = Mock(return_value=rvalue)
def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.session = json.loads(session.decode('utf-8'))
self.handler = SessionAnalyzer(loop=self.loop)
def __init__(self, loop=None):
self.sessions = []
self.analyzer = SessionAnalyzer(loop=loop)
self.logger = logging.getLogger(__name__)
def setUp(self):
self.session = json.loads(session.decode('utf-8'))
self.handler = SessionAnalyzer()
def setUp(self):
self.session = json.loads(session.decode('utf-8'))
self.handler = SessionAnalyzer()
class SessionManager:
def __init__(self):
self.sessions = []
self.analyzer = SessionAnalyzer()
self.logger = logging.getLogger(
'tanner.session_manager.SessionManager')
@asyncio.coroutine
def add_or_update_session(self, raw_data, redis_client):
# prepare the list of sessions
yield from self.delete_old_sessions(redis_client)
# handle raw data
valid_data = self.validate_data(raw_data)
# push snare uuid into redis.
yield from redis_client.sadd('snare_ids', [valid_data['uuid']])
session = self.get_session(valid_data)
if session is None:
try:
new_session = Session(valid_data)
except KeyError as key_error:
self.logger.error('Error during session creation: %s',
key_error)
return
self.sessions.append(new_session)
return new_session
else:
session.update_session(valid_data)
return session
@staticmethod
def validate_data(data):
if 'peer' not in data:
peer = dict(ip=None, port=None)
data['peer'] = peer
data['headers'] = dict(
(k.lower(), v) for k, v in data['headers'].items())
if 'user-agent' not in data['headers']:
data['headers']['user-agent'] = None
if 'path' not in data:
data['path'] = None
if 'uuid' not in data:
data['uuid'] = None
if 'status' not in data:
data['status'] = 200 if 'error' not in data else 500
return data
def get_session(self, data):
session = None
ip = data['peer']['ip']
user_agent = data['headers']['user-agent']
for sess in self.sessions:
if sess.ip == ip and sess.user_agent == user_agent:
session = sess
break
return session
@asyncio.coroutine
def delete_old_sessions(self, redis_client):
for sess in self.sessions:
if not sess.is_expired():
continue
sess.remove_associated_db()
self.sessions.remove(sess)
try:
yield from redis_client.set(sess.get_key(), sess.to_json())
yield from self.analyzer.analyze(sess.get_key(), redis_client)
except asyncio_redis.NotConnectedError as redis_error:
self.logger.error(
'Error connect to redis, session stay in memory. %s',
redis_error)
self.sessions.append(sess)
def __init__(self):
self.sessions = []
self.analyzer = SessionAnalyzer()
self.logger = logging.getLogger(
'tanner.session_manager.SessionManager')
class SessionManager:
def __init__(self):
self.sessions = []
self.analyzer = SessionAnalyzer()
self.logger = logging.getLogger('tanner.session_manager.SessionManager')
@asyncio.coroutine
def add_or_update_session(self, raw_data, redis_client):
# prepare the list of sessions
yield from self.delete_old_sessions(redis_client)
# handle raw data
valid_data = self.validate_data(raw_data)
# push snare uuid into redis.
yield from redis_client.sadd('snare_ids', [valid_data['uuid']])
session = self.get_session(valid_data)
if session is None:
try:
new_session = Session(valid_data)
except KeyError as key_error:
self.logger.error('Error during session creation: %s', key_error)
return
self.sessions.append(new_session)
return new_session
else:
session.update_session(valid_data)
return session
@staticmethod
def validate_data(data):
if 'peer' not in data:
peer = dict(ip=None, port=None)
data['peer'] = peer
data['headers'] = dict((k.lower(), v) for k, v in data['headers'].items())
if 'user-agent' not in data['headers']:
data['headers']['user-agent'] = None
if 'path' not in data:
data['path'] = None
if 'uuid' not in data:
data['uuid'] = None
if 'status' not in data:
data['status'] = 200 if 'error' not in data else 500
return data
def get_session(self, data):
session = None
ip = data['peer']['ip']
user_agent = data['headers']['user-agent']
for sess in self.sessions:
if sess.ip == ip and sess.user_agent == user_agent:
session = sess
break
return session
@asyncio.coroutine
def delete_old_sessions(self, redis_client):
for sess in self.sessions:
if not sess.is_expired():
continue
sess.remove_associated_db()
self.sessions.remove(sess)
try:
yield from redis_client.set(sess.get_key(), sess.to_json())
yield from self.analyzer.analyze(sess.get_key(), redis_client)
except asyncio_redis.NotConnectedError as redis_error:
self.logger.error('Error connect to redis, session stay in memory. %s', redis_error)
self.sessions.append(sess)
def __init__(self):
self.sessions = []
self.analyzer = SessionAnalyzer()
self.logger = logging.getLogger('tanner.session_manager.SessionManager')
def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.session = json.loads(session.decode('utf-8'))
self.handler = SessionAnalyzer(loop=self.loop)
self.res = None
geoip2.database.Reader.__init__ = Mock(return_value=None)
rvalue = geoip2.models.City(
{
'city': {
'geoname_id': 4223379,
'names': {
'en': 'Smyrna',
'ru': 'Смирна',
'zh-CN': '士麦那'
}
},
'continent': {
'code': 'NA',
'geoname_id': 6255149,
'names': {
'de': 'Nordamerika',
'en': 'North America',
'es': 'Norteamérica',
'fr': 'Amérique du Nord',
'ja': '北アメリカ',
'pt-BR': 'América do Norte',
'ru': 'Северная Америка',
'zh-CN': '北美洲'
}
},
'country': {
'geoname_id': 6252001,
'iso_code': 'US',
'names': {
'de': 'USA',
'en': 'United States',
'es': 'Estados Unidos',
'fr': 'États-Unis',
'ja': 'アメリカ合衆国',
'pt-BR': 'Estados Unidos',
'ru': 'США',
'zh-CN': '美国'
}
},
'location': {
'accuracy_radius': 10,
'latitude': 33.8633,
'longitude': -84.4984,
'metro_code': 524,
'time_zone': 'America/New_York'
},
'postal': {
'code': '30080'
},
'registered_country': {
'geoname_id': 6252001,
'iso_code': 'US',
'names': {
'de': 'USA',
'en': 'United States',
'es': 'Estados Unidos',
'fr': 'États-Unis',
'ja': 'アメリカ合衆国',
'pt-BR': 'Estados Unidos',
'ru': 'США',
'zh-CN': '美国'
}
},
'subdivisions': [{
'geoname_id': 4197000,
'iso_code': 'GA',
'names': {
'en': 'Georgia',
'es': 'Georgia',
'fr': 'Géorgie',
'ja': 'ジョージア州',
'pt-BR': 'Geórgia',
'ru': 'Джорджия',
'zh-CN': '乔治亚'
}
}],
'traits': {
'ip_address': '74.217.37.8'
}
}, ['en'])
geoip2.database.Reader.city = Mock(return_value=rvalue)
class TestSessionAnalyzer(unittest.TestCase):
def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.session = json.loads(session.decode('utf-8'))
self.handler = SessionAnalyzer(loop=self.loop)
self.res = None
geoip2.database.Reader.__init__ = Mock(return_value=None)
rvalue = geoip2.models.City(
{
'city': {
'geoname_id': 4223379,
'names': {
'en': 'Smyrna',
'ru': 'Смирна',
'zh-CN': '士麦那'
}
},
'continent': {
'code': 'NA',
'geoname_id': 6255149,
'names': {
'de': 'Nordamerika',
'en': 'North America',
'es': 'Norteamérica',
'fr': 'Amérique du Nord',
'ja': '北アメリカ',
'pt-BR': 'América do Norte',
'ru': 'Северная Америка',
'zh-CN': '北美洲'
}
},
'country': {
'geoname_id': 6252001,
'iso_code': 'US',
'names': {
'de': 'USA',
'en': 'United States',
'es': 'Estados Unidos',
'fr': 'États-Unis',
'ja': 'アメリカ合衆国',
'pt-BR': 'Estados Unidos',
'ru': 'США',
'zh-CN': '美国'
}
},
'location': {
'accuracy_radius': 10,
'latitude': 33.8633,
'longitude': -84.4984,
'metro_code': 524,
'time_zone': 'America/New_York'
},
'postal': {
'code': '30080'
},
'registered_country': {
'geoname_id': 6252001,
'iso_code': 'US',
'names': {
'de': 'USA',
'en': 'United States',
'es': 'Estados Unidos',
'fr': 'États-Unis',
'ja': 'アメリカ合衆国',
'pt-BR': 'Estados Unidos',
'ru': 'США',
'zh-CN': '美国'
}
},
'subdivisions': [{
'geoname_id': 4197000,
'iso_code': 'GA',
'names': {
'en': 'Georgia',
'es': 'Georgia',
'fr': 'Géorgie',
'ja': 'ジョージア州',
'pt-BR': 'Geórgia',
'ru': 'Джорджия',
'zh-CN': '乔治亚'
}
}],
'traits': {
'ip_address': '74.217.37.8'
}
}, ['en'])
geoip2.database.Reader.city = Mock(return_value=rvalue)
def tests_load_session_fail(self):
async def sess_get(key):
return aioredis.ProtocolError
redis_mock = Mock()
redis_mock.get = sess_get
res = None
with self.assertLogs():
self.loop.run_until_complete(self.handler.analyze(
None, redis_mock))
def test_create_stats(self):
async def sess_get():
return session
async def set_of_members(key):
return set()
async def set_add():
return ''
redis_mock = Mock()
redis_mock.get = sess_get
redis_mock.smembers = set_of_members
redis_mock.zadd = set_add
with patch('builtins.open', new_callable=mock_open) as m:
stats = self.loop.run_until_complete(
self.handler.create_stats(self.session, redis_mock))
self.assertEqual(stats['possible_owners'], {'attacker': 1.0})
def test_choose_owner_crawler(self):
stats = dict(paths=[{
'path': '/robots.txt',
'timestamp': 1.0,
'response_status': 200,
'attack_type': 'index'
}],
attack_types={'index'},
requests_in_second=11.1,
referer=None)
async def test():
self.res = await self.handler.choose_possible_owner(stats)
with patch('builtins.open', new_callable=mock_open) as m:
self.loop.run_until_complete(test())
self.assertEqual(self.res['possible_owners'], {'crawler': 1.0})
def test_choose_owner_attacker(self):
stats = dict(paths=[{
'path': '/',
'timestamp': 1.0,
'response_status': 200,
'attack_type': 'rfi'
}],
attack_types={'rfi', 'lfi'},
requests_in_second=2,
user_agent='user')
async def test():
self.res = await self.handler.choose_possible_owner(stats)
with patch('builtins.open', new_callable=mock_open) as m:
self.loop.run_until_complete(test())
self.assertEqual(self.res['possible_owners'], {'attacker': 1.0})
def test_choose_owner_mixed(self):
stats = dict(
paths=[{
'path': '/',
'timestamp': 1.0,
'response_status': 200,
'attack_type': ''
}],
attack_types='',
requests_in_second=2,
user_agent=
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
peer_ip='74.217.37.84',
hidden_links=0,
referer='/')
async def test():
self.res = await self.handler.choose_possible_owner(stats)
with patch('builtins.open', new_callable=mock_open) as m:
self.loop.run_until_complete(test())
self.assertEqual(self.res['possible_owners'], {
'attacker': 0.75,
'crawler': 0.25,
'tool': 0.15,
'user': 0.25
})
def test_choose_owner_user(self):
stats = dict(paths=[{
'path': '/',
'timestamp': 1.0,
'response_status': 200,
'attack_type': ''
}],
attack_types='',
requests_in_second=2,
user_agent='test_user_agent',
peer_ip='74.217.37.84',
hidden_links=0,
referer='/')
async def test():
self.res = await self.handler.choose_possible_owner(stats)
with patch('builtins.open', new_callable=mock_open) as m:
self.loop.run_until_complete(test())
self.assertEqual(self.res['possible_owners'], {'user': 1.0})
def test_find_location(self):
location_stats = self.handler.find_location("74.217.37.84")
expected_res = dict(
country='United States',
country_code='US',
city='Smyrna',
zip_code='30080',
)
self.assertEqual(location_stats, expected_res)
class TestSessionAnalyzer(unittest.TestCase):
def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
self.session = json.loads(session.decode('utf-8'))
self.handler = SessionAnalyzer(loop=self.loop)
self.res = None
geoip2.database.Reader.__init__ = Mock(return_value=None)
rvalue = geoip2.models.City(
{'city': {'geoname_id': 4223379, 'names': {'en': 'Smyrna',
'ru': 'Смирна', 'zh-CN': '士麦那'}},
'continent': {'code': 'NA', 'geoname_id': 6255149,
'names': {'de': 'Nordamerika', 'en': 'North America',
'es': 'Norteamérica', 'fr': 'Amérique du Nord',
'ja': '北アメリカ', 'pt-BR': 'América do Norte',
'ru': 'Северная Америка', 'zh-CN': '北美洲'}},
'country': {'geoname_id': 6252001, 'iso_code': 'US',
'names': {'de': 'USA', 'en': 'United States',
'es': 'Estados Unidos', 'fr': 'États-Unis',
'ja': 'アメリカ合衆国', 'pt-BR': 'Estados Unidos',
'ru': 'США', 'zh-CN': '美国'}},
'location': {'accuracy_radius': 10,
'latitude': 33.8633,
'longitude': -84.4984,
'metro_code': 524,
'time_zone': 'America/New_York'},
'postal': {'code': '30080'},
'registered_country': {'geoname_id': 6252001, 'iso_code': 'US',
'names': {'de': 'USA', 'en': 'United States',
'es': 'Estados Unidos', 'fr': 'États-Unis',
'ja': 'アメリカ合衆国', 'pt-BR': 'Estados Unidos',
'ru': 'США', 'zh-CN': '美国'}},
'subdivisions': [{'geoname_id': 4197000, 'iso_code': 'GA',
'names': {'en': 'Georgia', 'es': 'Georgia',
'fr': 'Géorgie', 'ja': 'ジョージア州',
'pt-BR': 'Geórgia', 'ru': 'Джорджия',
'zh-CN': '乔治亚'}}],
'traits': {'ip_address': '74.217.37.8'}}, ['en']
)
geoip2.database.Reader.city = Mock(return_value=rvalue)
def tests_load_session_fail(self):
async def sess_get(key):
return aioredis.ProtocolError
redis_mock = Mock()
redis_mock.get = sess_get
res = None
with self.assertLogs():
self.loop.run_until_complete(self.handler.analyze(None, redis_mock))
def test_create_stats(self):
async def sess_get():
return session
async def set_of_members(key):
return set()
async def set_add():
return ''
redis_mock = Mock()
redis_mock.get = sess_get
redis_mock.smembers = set_of_members
redis_mock.zadd = set_add
with patch('builtins.open', new_callable=mock_open) as m:
stats = self.loop.run_until_complete(self.handler.create_stats(self.session, redis_mock))
self.assertEqual(stats['possible_owners'], {'attacker': 1.0})
def test_choose_owner_crawler(self):
stats = dict(
paths=[{
'path': '/robots.txt',
'timestamp': 1.0, 'response_status': 200,
'attack_type': 'index'
}],
attack_types={'index'},
requests_in_second=11.1,
referer=None
)
async def test():
self.res = await self.handler.choose_possible_owner(stats)
with patch('builtins.open', new_callable=mock_open) as m:
self.loop.run_until_complete(test())
self.assertEqual(self.res['possible_owners'], {'crawler': 1.0})
def test_choose_owner_attacker(self):
stats = dict(
paths=[{
'path': '/',
'timestamp': 1.0, 'response_status': 200,
'attack_type': 'rfi'
}],
attack_types={'rfi', 'lfi'},
requests_in_second=2,
user_agent='user'
)
async def test():
self.res = await self.handler.choose_possible_owner(stats)
with patch('builtins.open', new_callable=mock_open) as m:
self.loop.run_until_complete(test())
self.assertEqual(self.res['possible_owners'], {'attacker': 1.0})
def test_choose_owner_mixed(self):
stats = dict(
paths=[{
'path': '/',
'timestamp': 1.0, 'response_status': 200,
'attack_type': ''
}],
attack_types='',
requests_in_second=2,
user_agent='Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
peer_ip='74.217.37.84',
hidden_links=0,
referer='/'
)
async def test():
self.res = await self.handler.choose_possible_owner(stats)
with patch('builtins.open', new_callable=mock_open) as m:
self.loop.run_until_complete(test())
self.assertEqual(self.res['possible_owners'], {'attacker': 0.75, 'crawler': 0.25, 'tool': 0.15, 'user': 0.25})
def test_choose_owner_user(self):
stats = dict(
paths=[{
'path': '/',
'timestamp': 1.0, 'response_status': 200,
'attack_type': ''
}],
attack_types='',
requests_in_second=2,
user_agent='test_user_agent',
peer_ip='74.217.37.84',
hidden_links=0,
referer='/'
)
async def test():
self.res = await self.handler.choose_possible_owner(stats)
with patch('builtins.open', new_callable=mock_open) as m:
self.loop.run_until_complete(test())
self.assertEqual(self.res['possible_owners'], {'user': 1.0})
def test_find_location(self):
location_stats = self.handler.find_location("74.217.37.84")
expected_res = dict(
country='United States',
country_code='US',
city='Smyrna',
zip_code='30080',
)
self.assertEqual(location_stats, expected_res)
