def test_worker_award(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.client.request.side_effect = [ResponseMock({'X-Request-ID': self.request_ids[0]}, munchify({'prev_page': {'offset': '123'}, 'next_page': {'offset': '1234'}, 'data': {'status': "active.pre-qualification", 'id': self.tender_id, 'procurementMethodType': 'aboveThresholdEU', 'awards': [ self.awards(0, 0, 'pending', CODES[0]), self.awards(1, 1, 'pending', CODES[1]), self.awards(2, 2, 'pending', CODES[2]), self.awards(3, 3, 'unsuccessful', CODES[2]), {'id': self.bid_ids[4], 'tenderers': [{'identifier': { 'scheme': 'UA-ED', 'id': CODES[2]}}]}]}}))] for i in range(3): data = Data(self.tender_id, self.award_ids[i], CODES[i], 'awards', {'meta': {'sourceRequests': [self.request_ids[0]]}}) self.check_data_objects(self.edrpou_codes_queue.get(), data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_ids[0]), item_key(self.tender_id, self.award_ids[1]), item_key(self.tender_id, self.award_ids[2])])
def test_update_processing_items(self): self.process_tracker.processing_items = { item_key(self.tender_id, self.item_id): 2 } self.assertEqual(self.process_tracker.processing_items, {item_key(self.tender_id, self.item_id): 2}) self.process_tracker.update_processing_items(self.tender_id, self.item_id) self.assertEqual(self.process_tracker.processing_items, {item_key(self.tender_id, self.item_id): 1}) self.process_tracker.update_processing_items(self.tender_id, self.item_id) self.assertEqual(self.process_tracker.processing_items, {})
def test_filtered_tender_ids_queue_loop_exit(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep filtered_tender_ids_queue = MagicMock() filtered_tender_ids_queue.peek.side_effect = [ LoopExit(), self.tender_id ] self.client.request.return_value = ResponseMock( {'X-Request-ID': self.request_ids[0]}, munchify({ 'prev_page': { 'offset': '123' }, 'next_page': { 'offset': '1234' }, 'data': { 'status': "active.pre-qualification", 'id': self.tender_id, 'procurementMethodType': 'aboveThresholdEU', 'awards': [self.awards(0, 0, 'pending', CODES[0])] } })) first_data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards', {'meta': { 'sourceRequests': [self.request_ids[0]] }}) self.worker.filtered_tender_ids_queue = filtered_tender_ids_queue self.check_data_objects(self.edrpou_codes_queue.get(), first_data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_ids[0])])
def test_retry_upload_to_tender(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep mrequest.post(self.url, json=self.stat_200(), status_code=200) self.client._create_tender_resource_item.side_effect = [ Unauthorized(http_code=401), Unauthorized(http_code=403), Unauthorized(http_code=429), self.get_tender() ] self.upload_to_doc_service_queue.put(self.data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1) self.shutdown_when_done(self.worker) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(mrequest.call_count, 1) self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') self.assertIsNotNone( mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertEqual(self.process_tracker.processing_items, {}) # test that item removed from processing_items self.assertEqual(self.client._create_tender_resource_item.call_count, 4) # check upload to tender
def test_retry_doc_service(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') mrequest.post(self.url, [{ 'text': '', 'status_code': 401 } for _ in range(6)] + [{ 'json': { 'data': { 'url': 'test url', 'format': 'application/yaml', 'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555', 'title': file_name } }, 'status_code': 200 }]) self.upload_to_doc_service_queue.put(self.data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1) self.shutdown_when_done(self.worker) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.upload_to_tender_queue.qsize(), 1, 'Queue should be have 1 element') self.assertEqual(mrequest.call_count, 7) self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') self.assertIsNotNone( mrequest.request_history[0].headers['X-Client-Request-ID'])
def test_get_tender_429(self, gevent_sleep): """ We must not lose tender after restart filter worker """ gevent_sleep.side_effect = custom_sleep self.client.request.side_effect = [ ResourceError(http_code=429), ResponseMock({'X-Request-ID': self.request_ids[0]}, munchify({ 'prev_page': { 'offset': '123' }, 'next_page': { 'offset': '1234' }, 'data': { 'status': "active.pre-qualification", 'id': self.tender_id, 'procurementMethodType': 'aboveThresholdEU', 'awards': [self.awards(0, 0, 'pending', CODES[0])] } })) ] data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards', {'meta': { 'sourceRequests': [self.request_ids[0]] }}) self.sleep_change_value.increment_step = 2 self.sleep_change_value.decrement_step = 1 self.check_data_objects(self.edrpou_codes_queue.get(), data) self.assertEqual(self.worker.sleep_change_value.time_between_requests, 1) gevent_sleep.assert_called_with_once(1) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_ids[0])]) self.assertEqual(self.edrpou_codes_queue.qsize(), 0)
def test_worker_dead(self, gevent_sleep): """ Test that worker will process tender after exception """ gevent_sleep.side_effect = custom_sleep self.filtered_tender_ids_queue.put(self.tender_id) self.client.request.side_effect = [ ResponseMock({'X-Request-ID': self.request_ids[i]}, munchify({ 'prev_page': { 'offset': '123' }, 'next_page': { 'offset': '1234' }, 'data': { 'status': "active.pre-qualification", 'id': self.tender_id, 'procurementMethodType': 'aboveThresholdEU', 'awards': [self.awards(i, i, 'pending', CODES[0])] } })) for i in range(2) ] for i in range(2): data = Data(self.tender_id, self.award_ids[i], CODES[0], 'awards', {'meta': { 'sourceRequests': [self.request_ids[i]] }}) self.check_data_objects(self.edrpou_codes_queue.get(), data) self.worker.job.kill(timeout=1) self.assertItemsEqual( self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_ids[i]) for i in range(2)])
def test_worker_award_with_cancelled_lot(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.client.request.return_value = ResponseMock({'X-Request-ID': self.request_ids[0]}, munchify({'prev_page': {'offset': '123'}, 'next_page': {'offset': '1234'}, 'data': {'status': "active.pre-qualification", 'id': self.tender_id, 'procurementMethodType': 'aboveThresholdEU', 'lots': [{'status': 'cancelled', 'id': '123456789'}, {'status': 'active', 'id': '12345678'}], 'awards': [{'id': self.award_ids[0], 'bid_id': self.bid_ids[0], 'status': 'pending', 'suppliers': [{'identifier': { 'scheme': 'UA-EDR', 'id': CODES[0]}}], 'lotID': '123456789'}, {'id': self.award_ids[1], 'bid_id': self.bid_ids[1], 'status': 'pending', 'suppliers': [{'identifier': { 'scheme': 'UA-EDR', 'id': CODES[1]}}], 'lotID': '12345678'}]}})) data = Data(self.tender_id, self.award_ids[1], CODES[1], 'awards', {'meta': {'sourceRequests': [self.request_ids[0]]}}) for edrpou in [data]: self.check_data_objects(self.edrpou_codes_queue.get(), edrpou) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_ids[1])])
def test_successfully_uploaded_to_tender_retry(self): self.worker.retry_upload_to_tender_queue.put(self.data) self.assertEqual(self.worker.process_tracker.processing_items, {item_key(self.tender_id, self.award_id): 1}) self.worker.successfully_uploaded_to_tender(self.data, True) self.assertEqual(self.worker.retry_upload_to_tender_queue.qsize(), 0) self.assertEqual(self.worker.process_tracker.processing_items, {})
def test_set_item(self): self.assertEqual(self.process_tracker.processing_items, {}) self.assertEqual(self.process_tracker.tender_documents_to_process, {}) self.process_tracker.set_item(self.tender_id, self.item_id, 1) self.assertEqual(self.process_tracker.processing_items, {item_key(self.tender_id, self.item_id): 1}) self.assertEqual(self.process_tracker.tender_documents_to_process, {db_key(self.tender_id): 1})
def _update_processing_items(self, tender_id, item_id, document_id): key = item_key(tender_id, item_id) if self.processing_items[key] > 1: self.processing_items[key] -= 1 else: self.processed_items[key] = datetime.now() self._remove_unprocessed_item(document_id) del self.processing_items[key]
def test_successful_upload(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep mrequest.post(self.url, json=self.stat_200(), status_code=200) self.upload_to_doc_service_queue.put(self.data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1) self.shutdown_when_done(self.worker) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.upload_to_tender_queue.qsize(), 1, 'Queue should be have 1 element') self.assertEqual(mrequest.call_count, 1) self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') self.assertIsNotNone( mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)])
def test_filtered_tender_ids_queue_loop_exit(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep filtered_tender_ids_queue = MagicMock() filtered_tender_ids_queue.peek.side_effect = [LoopExit(), self.tender_id] self.client.request.return_value = self.response first_data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards', {'meta': {'sourceRequests': [self.request_ids[0]]}}) self.worker.filtered_tender_ids_queue = filtered_tender_ids_queue self.check_data_objects(self.edrpou_codes_queue.get(), first_data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_ids[0])])
def test_get_tender_exception(self, gevent_sleep): """ We must not lose tender after restart filter worker """ gevent_sleep.side_effect = custom_sleep self.client.request.side_effect = [Exception(), self.response] data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards', {'meta': {'sourceRequests': [self.request_ids[0]]}}) self.check_data_objects(self.edrpou_codes_queue.get(), data) self.assertEqual(self.worker.sleep_change_value.time_between_requests, 0) gevent_sleep.assert_called_with_once(1) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_ids[0])]) self.assertEqual(self.edrpou_codes_queue.qsize(), 0)
def test_request_failed_item_status_change(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep mrequest.post(self.url, json=self.stat_200(), status_code=200) self.process_tracker.set_item(self.tender_id, self.qualification_id, 1) self.upload_to_doc_service_queue.put(self.data) self.upload_to_doc_service_queue.put(self.qualification_data) self.shutdown_when_done(self.worker) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.upload_to_tender_queue.get(), self.data) self.assertEqual(self.upload_to_tender_queue.get(), self.qualification_data) self.assertEqual(mrequest.call_count, 2) self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') self.assertIsNotNone( mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertEqual( self.process_tracker.processing_items, { item_key(self.tender_id, self.award_id): 1, item_key(self.tender_id, self.qualification_id): 1 })
def test_412(self, mrequest, gevent_sleep): gevent_sleep.side_effect = custom_sleep mrequest.post(self.url, json=self.stat_200(), status_code=200) api_server_bottle = Bottle() api_server = WSGIServer(('127.0.0.1', 20604), api_server_bottle, log=None) setup_routing(api_server_bottle, response_spore) setup_routing(api_server_bottle, response_412, path='/api/2.3/tenders/{}/awards/{}/documents'.format( self.tender_id, self.award_id), method='POST') api_server.start() self.worker.client = TendersClientSync( '', host_url='http://127.0.0.1:20604', api_version='2.3') setup_routing(api_server_bottle, generate_response, path='/api/2.3/tenders/{}/awards/{}/documents'.format( self.tender_id, self.award_id), method='POST') self.assertEqual( self.worker.client.headers['Cookie'], 'SERVER_ID={}'.format( SPORE_COOKIES)) # check that response_spore set cookies self.upload_to_doc_service_queue.put(self.data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1) self.worker.start() self.shutdown_when_done(self.worker) self.assertEqual( self.worker.client.headers['Cookie'], 'SERVER_ID={}'.format( COOKIES_412)) # check that response_412 change cookies self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(mrequest.call_count, 1) self.assertEqual(mrequest.request_history[0].url, u'127.0.0.1:80/upload') self.assertIsNotNone( mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [])
def test_retry_upload_to_tender(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep self.client._create_tender_resource_item.side_effect = [ Unauthorized(http_code=401), Unauthorized(http_code=403), Unauthorized(http_code=429), self.get_tender() ] self.upload_to_tender_queue.put(self.data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) self.assertEqual(self.upload_to_tender_queue.qsize(), 1) self.shutdown_when_done(self.worker) self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.process_tracker.processing_items, {}) # test that item removed from processing_items self.assertEqual(self.client._create_tender_resource_item.call_count, 4) # check upload to tender
def test_worker_restart(self, gevent_sleep): """ Process tender after catch Unauthorized exception """ gevent_sleep.side_effect = custom_sleep self.client.request.side_effect = [ Unauthorized(http_code=403), Unauthorized(http_code=403), Unauthorized(http_code=403), ResponseMock({'X-Request-ID': self.request_ids[0]}, munchify({'prev_page': {'offset': '123'}, 'next_page': {'offset': '1234'}, 'data': {'status': "active.pre-qualification", 'id': self.tender_id, 'procurementMethodType': 'aboveThresholdEU', 'awards': [self.awards(0, 0, 'pending', CODES[0]), self.awards(1, 1, 'unsuccessful', CODES[2])]}}))] data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards', {'meta': {'sourceRequests': [self.request_ids[0]]}}) self.check_data_objects(self.edrpou_codes_queue.get(), data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_ids[0])])
def test_upload_to_doc_service_queue_loop_exit(self, mrequest, gevent_sleep): """ Test LoopExit for upload_to_doc_service_queue """ gevent_sleep.side_effect = custom_sleep self.process_tracker.set_item(self.tender_id, self.award_id, 2) self.worker.upload_to_doc_service_queue = MagicMock() self.worker.upload_to_doc_service_queue.peek.side_effect = generate_answers( answers=[LoopExit(), self.data, self.data], default=LoopExit()) mrequest.post(self.url, [{ 'json': self.stat_200(), 'status_code': 200 } for _ in range(2)]) self.worker.start() sleep(1) self.assertEqual(self.upload_to_tender_queue.get(), self.data) self.assertIsNotNone( mrequest.request_history[0].headers['X-Client-Request-ID']) self.assertIsNotNone( mrequest.request_history[1].headers['X-Client-Request-ID']) self.assertEqual(self.process_tracker.processing_items, {item_key(self.tender_id, self.award_id): 2})
def test_process_412(self, gevent_sleep): gevent_sleep.side_effect = custom_sleep api_server_bottle = Bottle() api_server = WSGIServer(('127.0.0.1', 20604), api_server_bottle, log=None) setup_routing(api_server_bottle, response_spore) setup_routing(api_server_bottle, response_412, path='/api/2.3/tenders/{}/awards/{}/documents'.format( self.tender_id, self.award_id), method='POST') api_server.start() self.worker.client = TendersClient('', host_url='http://127.0.0.1:20604', api_version='2.3') setup_routing(api_server_bottle, generate_response, path='/api/2.3/tenders/{}/awards/{}/documents'.format( self.tender_id, self.award_id), method='POST') self.assertEqual(self.worker.client.headers['Cookie'], 'SERVER_ID={}'.format(SPORE_COOKIES)) self.upload_to_tender_queue.put(self.data) self.assertItemsEqual(self.process_tracker.processing_items.keys(), [item_key(self.tender_id, self.award_id)]) self.assertEqual(self.upload_to_tender_queue.qsize(), 1) self.shutdown_when_done(self.worker) self.assertEqual(self.worker.client.headers['Cookie'], 'SERVER_ID={}'.format(COOKIES_412)) self.assertEqual(self.upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertEqual(self.worker.retry_upload_to_tender_queue.qsize(), 0, 'Queue should be empty') self.assertItemsEqual(self.process_tracker.processing_items.keys(), []) api_server.stop()
def set_item(self, tender_id, item_id, docs_amount=0): self.processing_items[item_key(tender_id, item_id)] = docs_amount self._add_docs_amount_to_tender(tender_id, docs_amount)
def check_processed_item(self, tender_id, item_id): """Check if current tender_id, item_id was already processed""" return item_key(tender_id, item_id) in self.processed_items.keys()
def check_processing_item(self, tender_id, item_id): """Check if current tender_id, item_id is processing""" return item_key(tender_id, item_id) in self.processing_items.keys()