def test_perfomance_watcher(self): bridge = BasicDataBridge(self.config) for i in xrange(0, 3): bridge.create_api_client() req_duration = 1 for _, info in bridge.api_clients_info.items(): info['request_durations'][datetime.datetime.now()] = req_duration req_duration += 1 self.assertEqual(info.get('grown', False), False) self.assertEqual(len(info['request_durations']), 1) self.assertEqual(len(bridge.api_clients_info), 3) self.assertEqual(bridge.api_clients_queue.qsize(), 3) sleep(1) bridge.perfomance_watcher() grown = 0 with_new_cookies = 0 for cid, info in bridge.api_clients_info.items(): if info.get('grown', False): grown += 1 if info['drop_cookies']: with_new_cookies += 1 self.assertEqual(len(info['request_durations']), 0) self.assertEqual(len(bridge.api_clients_info), 3) self.assertEqual(bridge.api_clients_queue.qsize(), 3) self.assertEqual(grown, 3) self.assertEqual(with_new_cookies, 1)
def test__mark_bad_clients(self): bridge = BasicDataBridge(self.config) self.assertEqual(bridge.api_clients_queue.qsize(), 0) self.assertEqual(len(bridge.api_clients_info), 0) bridge.create_api_client() bridge.create_api_client() bridge.create_api_client() self.assertEqual(len(bridge.api_clients_info), 3) avg_duration = 1 req_intervals = [0, 2, 0, 0] for cid in bridge.api_clients_info: self.assertEqual(bridge.api_clients_info[cid]['drop_cookies'], False) bridge.api_clients_info[cid]['avg_duration'] = avg_duration bridge.api_clients_info[cid]['grown'] = True bridge.api_clients_info[cid]['request_interval'] = \ req_intervals[avg_duration] avg_duration += 1 avg = 1.5 bridge._mark_bad_clients(avg) self.assertEqual(len(bridge.api_clients_info), 3) self.assertEqual(bridge.api_clients_queue.qsize(), 3) to_destroy = 0 for cid in bridge.api_clients_info: if bridge.api_clients_info[cid]['drop_cookies']: to_destroy += 1 self.assertEqual(to_destroy, 3)
def test_create_api_client(self, mock_APIClient): mock_APIClient.side_effect = [ RequestFailed(), munchify({'session': { 'headers': { 'User-Agent': 'test.agent' } }}) ] bridge = BasicDataBridge(self.config) self.assertEqual(bridge.api_clients_queue.qsize(), 0) bridge.create_api_client() self.assertEqual(bridge.api_clients_queue.qsize(), 1) del bridge
def test__get_average_request_duration(self, mocked_session): mocked_session.request.return_value = MockedResponse(200) bridge = BasicDataBridge(self.config) bridge.create_api_client() bridge.create_api_client() bridge.create_api_client() res, _ = bridge._get_average_requests_duration() self.assertEqual(res, 0) request_duration = 1 for k in bridge.api_clients_info: for i in xrange(0, 3): bridge.api_clients_info[k]['request_durations'][ datetime.datetime.now()] = request_duration request_duration += 1 res, res_list = bridge._get_average_requests_duration() self.assertEqual(res, 2) self.assertEqual(len(res_list), 3) delta = datetime.timedelta(seconds=301) grown_date = datetime.datetime.now() - delta bridge.api_clients_info[uuid.uuid4().hex] = { 'request_durations': { grown_date: 1 }, 'destroy': False, 'request_interval': 0, 'avg_duration': 0 } self.assertEqual(len(bridge.api_clients_info), 4) res, res_list = bridge._get_average_requests_duration() grown = 0 for k in bridge.api_clients_info: if bridge.api_clients_info[k].get('grown', False): grown += 1 self.assertEqual(res, 1.75) self.assertEqual(len(res_list), 4) self.assertEqual(grown, 1)