def test_request_failed_in_retry_item_status(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.client._create_tender_resource_item.side_effect = [ ResourceError(http_code=429) ] + [ResourceError(http_code=403) for _ in range(4)] self.worker.retry_upload_to_tender_queue.put(self.data) self.shutdown_when_done(self.worker) self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty')
def test_retry_upload_to_tender_429(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.client.client_upload_to_tender = MagicMock(side_effect=[ ResourceError(http_code=429), ResourceError(http_code=403) ]) self.worker.retry_upload_to_tender_queue = Queue(10) self.worker.retry_upload_to_tender_queue.put(self.data) self.shutdown_when_done(self.worker) self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty')
def test_upload_to_tender_429(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.client._create_tender_resource_item = MagicMock(side_effect=[ ResourceError(http_code=429), ResourceError(http_code=429), ResourceError(http_code=403) ]) self.upload_to_tender_queue.put(self.data) self.shutdown_when_done(self.worker) self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.worker.sleep_change_value.time_between_requests, 1)
def test_check_412_function(self): func = check_412( MagicMock(side_effect=ResourceError( http_code=412, response=MagicMock(headers={'Set-Cookie': 1})))) with self.assertRaises(ResourceError): func(MagicMock(headers={'Cookie': 1})) func = check_412( MagicMock(side_effect=ResourceError( http_code=403, response=MagicMock(headers={'Set-Cookie': 1})))) with self.assertRaises(ResourceError): func(MagicMock(headers={'Cookie': 1})) f = check_412(MagicMock(side_effect=[1])) self.assertEqual(f(1), 1)
def test_request_failed_in_retry(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.worker.do_upload_to_tender_with_retry = MagicMock() self.worker.do_upload_to_tender_with_retry.side_effect = [ ResourceError(http_code=429) for _ in range(5) ] + [ResourceError(http_code=403)] self.sleep_change_value.increment_step = 3 self.sleep_change_value.decrement_step = 1.5 self.worker.retry_upload_to_tender_queue.put(self.data) self.shutdown_when_done(self.worker) self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.worker.sleep_change_value.time_between_requests, 13.5)
def test_request_failed_in_retry_item_status(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep mrequest.post(self.url, json=self.stat_200(), status_code=200) self.client._create_tender_resource_item.side_effect = [ ResourceError(http_code=429) ] + [ResourceError(http_code=403) for _ in range(4)] self.worker.retry_upload_to_tender_queue.put(self.data) self.shutdown_when_done(self.worker) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.process_tracker.processing_items, {}) self.assertEqual(self.client._create_tender_resource_item.call_count, 5) # check that processed just 1 request
def test_try_upload_to_tender_resource_error(self): re = ResourceError("test resource error") self.worker.update_headers_and_upload_to_tender = MagicMock( side_effect=re) self.worker.remove_data_or_increase_wait = MagicMock() self.worker.try_upload_to_tender(self.data, False) self.worker.remove_data_or_increase_wait.assert_called_once_with( re, self.data, False)
def test_handle_error(self): re = ResourceError("error", 404) self.worker.upload_to_tender_queue.put(self.data) self.worker.handle_error(re, self.data, False) self.assertEqual(self.worker.upload_to_tender_queue.qsize(), 0) self.assertEqual(self.worker.retry_upload_to_tender_queue.get(), self.data) self.assertEqual(self.worker.retry_upload_to_tender_queue.qsize(), 0)
def test_retry_upload_to_tender_exception(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.worker.do_upload_to_tender_with_retry = MagicMock( side_effect=[Exception(), ResourceError(http_code=403)]) self.worker.retry_upload_to_tender_queue.put(self.data) self.shutdown_when_done(self.worker) self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty')
def test_removing_data(self): re = ResourceError("error") self.worker.sleep_change_value.time_between_requests = 1 self.worker.upload_to_tender_queue.put(self.data) self.worker.removing_data(re, self.data, False) self.assertEqual(self.worker.process_tracker.processing_items, {}) self.assertEqual(self.worker.upload_to_tender_queue.qsize(), 0) self.assertEqual(self.worker.sleep_change_value.time_between_requests, 0)
def func_wrapper(obj, *args, **kwargs): try: response = func(obj, *args, **kwargs) except ResourceError as re: if re.status_int == 412: obj.headers['Cookie'] = re.response.headers['Set-Cookie'] response = func(obj, *args, **kwargs) else: raise ResourceError(re) return response
def test_request_failed_in_retry(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.worker.client_upload_to_tender = MagicMock() self.worker.client_upload_to_tender.side_effect = [ ResourceError(http_code=429) for _ in range(5) ] + [ResourceError(http_code=403)] self.sleep_change_value.increment_step = 3 self.sleep_change_value.decrement_step = 1.5 self.worker.retry_upload_to_tender_queue.put(self.data) self.shutdown_when_done(self.worker) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.worker.retry_upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.worker.sleep_change_value.time_between_requests, 13.5) self.assertEqual(self.process_tracker.processing_items, {}) self.assertEqual(self.worker.client_upload_to_tender.call_count, 6) # check that processed just 1 request
def test_request_failed(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep mrequest.post(self.url, json=self.stat_200(), status_code=200) self.client._create_tender_resource_item.side_effect = ResourceError( http_code=422) self.upload_to_doc_service_queue.put(self.data) self.shutdown_when_done(self.worker) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(mrequest.call_count, 1) self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') self.assertIsNotNone( mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertEqual(self.process_tracker.processing_items, {}) self.assertEqual(self.client._create_tender_resource_item.call_count, 1) # check that processed just 1 request
def test_decrease_request_frequency(self): re = ResourceError("error", 429) self.worker.decrease_request_frequency(re, self.data) self.assertEqual(self.worker.sleep_change_value.time_between_requests, 1)
def test_remove_data_or_increase_wait(self): re = ResourceError("error") self.worker.removing_data = MagicMock() self.worker.remove_data_or_increase_wait(re, self.data, False) self.worker.removing_data.assert_called_once_with(re, self.data, False)
def test_remove_data_or_increase_wait_else(self): re = ResourceError("error", http_code=404) self.worker.handle_error = MagicMock() self.worker.remove_data_or_increase_wait(re, self.data, False) self.worker.handle_error.assert_called_once_with(re, self.data, False)
def test_remove_data_or_increase_wait_429(self): re = ResourceError("error", http_code=429) self.worker.decrease_request_frequency = MagicMock() self.worker.remove_data_or_increase_wait(re, self.data, False) self.worker.decrease_request_frequency.assert_called_once_with( re, self.data)