def test_perfomance_watcher(self):
bridge = BasicDataBridge(self.config)
for i in xrange(0, 3):
bridge.create_api_client()
req_duration = 1
for _, info in bridge.api_clients_info.items():
info['request_durations'][datetime.datetime.now()] = req_duration
req_duration += 1
self.assertEqual(info.get('grown', False), False)
self.assertEqual(len(info['request_durations']), 1)
self.assertEqual(len(bridge.api_clients_info), 3)
self.assertEqual(bridge.api_clients_queue.qsize(), 3)
sleep(1)
bridge.perfomance_watcher()
grown = 0
with_new_cookies = 0
for cid, info in bridge.api_clients_info.items():
if info.get('grown', False):
grown += 1
if info['drop_cookies']:
with_new_cookies += 1
self.assertEqual(len(info['request_durations']), 0)
self.assertEqual(len(bridge.api_clients_info), 3)
self.assertEqual(bridge.api_clients_queue.qsize(), 3)
self.assertEqual(grown, 3)
self.assertEqual(with_new_cookies, 1)
def test__mark_bad_clients(self):
bridge = BasicDataBridge(self.config)
self.assertEqual(bridge.api_clients_queue.qsize(), 0)
self.assertEqual(len(bridge.api_clients_info), 0)
bridge.create_api_client()
bridge.create_api_client()
bridge.create_api_client()
self.assertEqual(len(bridge.api_clients_info), 3)
avg_duration = 1
req_intervals = [0, 2, 0, 0]
for cid in bridge.api_clients_info:
self.assertEqual(bridge.api_clients_info[cid]['drop_cookies'],
False)
bridge.api_clients_info[cid]['avg_duration'] = avg_duration
bridge.api_clients_info[cid]['grown'] = True
bridge.api_clients_info[cid]['request_interval'] = \
req_intervals[avg_duration]
avg_duration += 1
avg = 1.5
bridge._mark_bad_clients(avg)
self.assertEqual(len(bridge.api_clients_info), 3)
self.assertEqual(bridge.api_clients_queue.qsize(), 3)
to_destroy = 0
for cid in bridge.api_clients_info:
if bridge.api_clients_info[cid]['drop_cookies']:
to_destroy += 1
self.assertEqual(to_destroy, 3)
def test_create_api_client(self, mock_APIClient):
mock_APIClient.side_effect = [
RequestFailed(),
munchify({'session': {
'headers': {
'User-Agent': 'test.agent'
}
}})
]
bridge = BasicDataBridge(self.config)
self.assertEqual(bridge.api_clients_queue.qsize(), 0)
bridge.create_api_client()
self.assertEqual(bridge.api_clients_queue.qsize(), 1)
del bridge
def test__get_average_request_duration(self, mocked_session):
mocked_session.request.return_value = MockedResponse(200)
bridge = BasicDataBridge(self.config)
bridge.create_api_client()
bridge.create_api_client()
bridge.create_api_client()
res, _ = bridge._get_average_requests_duration()
self.assertEqual(res, 0)
request_duration = 1
for k in bridge.api_clients_info:
for i in xrange(0, 3):
bridge.api_clients_info[k]['request_durations'][
datetime.datetime.now()] = request_duration
request_duration += 1
res, res_list = bridge._get_average_requests_duration()
self.assertEqual(res, 2)
self.assertEqual(len(res_list), 3)
delta = datetime.timedelta(seconds=301)
grown_date = datetime.datetime.now() - delta
bridge.api_clients_info[uuid.uuid4().hex] = {
'request_durations': {
grown_date: 1
},
'destroy': False,
'request_interval': 0,
'avg_duration': 0
}
self.assertEqual(len(bridge.api_clients_info), 4)
res, res_list = bridge._get_average_requests_duration()
grown = 0
for k in bridge.api_clients_info:
if bridge.api_clients_info[k].get('grown', False):
grown += 1
self.assertEqual(res, 1.75)
self.assertEqual(len(res_list), 4)
self.assertEqual(grown, 1)
