def test_worker_award_with_cancelled_lot(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.client.request.return_value = ResponseMock(
{'X-Request-ID': self.request_ids[0]},
munchify({'prev_page': {'offset': '123'},
'next_page': {'offset': '1234'},
'data': {'status': tender_status,
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'lots': [{'status': 'cancelled', 'id': '123456789'},
{'status': 'active', 'id': '12345678'}],
'awards': [{'id': self.award_ids[0],
'bid_id': self.bid_ids[0],
'status': AWARD_STATUS,
'suppliers': [{'identifier': {'scheme': 'UA-EDR', 'id': CODES[0],
"legalName": "company_name"}}],
'lotID': '12345678'},
{'id': self.award_ids[1],
'bid_id': self.bid_ids[1],
'status': 'cancelled',
'suppliers': [{'identifier': {'scheme': 'UA-EDR', 'id': CODES[1],
"legalName": "company_name"}}],
'lotID': '123456789'}]}}))
data = Data(self.tender_id, self.award_ids[0], CODES[0], "company_name",
{"meta": {"sourceRequests": [self.request_ids[0]]}})
self.assertEqual(self.edrpou_codes_queue.get(), data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0])])
def test_successfully_uploaded_to_tender_retry(self):
self.worker.retry_upload_to_tender_queue.put(self.data)
self.assertEqual(self.worker.process_tracker.processing_items,
{item_key(self.tender_id, self.award_id): 1})
self.worker.successfully_uploaded_to_tender(self.data, True)
self.assertEqual(self.worker.retry_upload_to_tender_queue.qsize(), 0)
self.assertEqual(self.worker.process_tracker.processing_items, {})
def test_retry_doc_service(self, mrequest, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
doc_service_client = DocServiceClient(host='127.0.0.1',
port='80',
user='',
password='')
mrequest.post(self.url, [{
'text': '',
'status_code': 401
} for _ in range(6)] + [{
'json': {
'data': {
'url': 'test url',
'format': 'application/yaml',
'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555',
'title': file_name
}
},
'status_code': 200
}])
self.upload_to_doc_service_queue.put(self.data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.upload_to_tender_queue.qsize(), 1,
'Queue should be have 1 element')
self.assertEqual(mrequest.call_count, 7)
self.assertEqual(mrequest.request_history[0].url,
u'127.0.0.1:80/upload')
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
def test_request_failed_item_status_change(self, mrequest, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
mrequest.post(self.url, json=self.stat_200(), status_code=200)
self.process_tracker.set_item(self.tender_id, self.qualification_id, 1)
self.upload_to_doc_service_queue.put(self.data)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.upload_to_tender_queue.get(), self.tender_data)
self.assertEqual(mrequest.call_count, 1)
self.assertEqual(mrequest.request_history[0].url,
u'127.0.0.1:80/upload')
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
self.assertEqual(
self.process_tracker.processing_items, {
item_key(self.tender_id, self.award_id): 1,
item_key(self.tender_id, self.qualification_id): 1
})
def test_successful_upload(self, mrequest, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
mrequest.post(self.url, json=self.stat_200(), status_code=200)
self.upload_to_doc_service_queue.put(self.data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.upload_to_tender_queue.qsize(), 1,
'Queue should be have 1 element')
self.assertEqual(mrequest.call_count, 1)
self.assertEqual(mrequest.request_history[0].url,
u'127.0.0.1:80/upload')
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
def test_filtered_tender_ids_queue_loop_exit(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
filtered_tender_ids_queue = MagicMock()
filtered_tender_ids_queue.peek.side_effect = [LoopExit(), self.tender_id]
self.client.request.return_value = self.response
first_data = Data(self.tender_id, self.award_ids[0], CODES[0], "company_name",
{"meta": {"sourceRequests": [self.request_ids[0]]}})
self.worker.filtered_tender_ids_queue = filtered_tender_ids_queue
self.assertEqual(self.edrpou_codes_queue.get(), first_data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0])])
def test_get_tender_exception(self, gevent_sleep):
""" We must not lose tender after restart filter worker """
gevent_sleep.side_effect = custom_sleep
self.client.request.side_effect = [Exception(), self.response]
data = Data(self.tender_id, self.award_ids[0], CODES[0], "company_name",
{"meta": {"sourceRequests": [self.request_ids[0]]}})
self.assertEqual(self.edrpou_codes_queue.get(), data)
self.assertEqual(self.worker.sleep_change_value.time_between_requests, 0)
gevent_sleep.assert_called_with_once(1)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0])])
self.assertEqual(self.edrpou_codes_queue.qsize(), 0)
def test_retry_upload_to_tender(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.client._create_tender_resource_item.side_effect = [
Unauthorized(http_code=401),
Unauthorized(http_code=403),
Unauthorized(http_code=429),
self.get_tender()
]
self.upload_to_tender_queue.put(self.data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
self.assertEqual(self.upload_to_tender_queue.qsize(), 1)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.process_tracker.processing_items,
{}) # test that item removed from processing_items
self.assertEqual(self.client._create_tender_resource_item.call_count,
4) # check upload to tender
def test_worker_dead(self, gevent_sleep):
""" Test that worker will process tender after exception """
gevent_sleep.side_effect = custom_sleep
self.filtered_tender_ids_queue.put(self.tender_id)
self.client.request.side_effect = [
ResponseMock({'X-Request-ID': self.request_ids[i]},
munchify({'prev_page': {'offset': '123'},
'next_page': {'offset': '1234'},
'data': {
'status': tender_status,
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'awards': [self.awards(i, i, AWARD_STATUS, CODES[0])]}})) for i in range(2)]
for i in range(2):
data = Data(self.tender_id, self.award_ids[i], CODES[0], "company_name",
{"meta": {"sourceRequests": [self.request_ids[i]]}})
self.assertEqual(self.edrpou_codes_queue.get(), data)
self.worker.immortal_jobs['prepare_data'].kill(timeout=1)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[i]) for i in range(2)])
def test_worker_restart(self, gevent_sleep):
""" Process tender after catch Unauthorized exception """
gevent_sleep.side_effect = custom_sleep
self.client.request.side_effect = [
Unauthorized(http_code=403),
Unauthorized(http_code=403),
Unauthorized(http_code=403),
ResponseMock({'X-Request-ID': self.request_ids[0]},
munchify({'prev_page': {'offset': '123'},
'next_page': {'offset': '1234'},
'data': {'status': tender_status,
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'awards': [self.awards(0, 0, AWARD_STATUS, CODES[0]),
self.awards(1, 1, 'unsuccessful', CODES[2])]}}))]
data = Data(self.tender_id, self.award_ids[0], CODES[0], "company_name",
{"meta": {"sourceRequests": [self.request_ids[0]]}})
self.assertEqual(self.edrpou_codes_queue.get(), data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0])])
def test_worker_award(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.client.request.side_effect = [
ResponseMock({'X-Request-ID': self.request_ids[0]},
munchify({'prev_page': {'offset': '123'},
'next_page': {'offset': '1234'},
'data': {'status': "active.pre-qualification",
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'awards': [
self.awards(0, 0, AWARD_STATUS, CODES[0]),
self.awards(3, 3, 'unsuccessful', CODES[2]),
{'id': self.bid_ids[4],
'tenderers': [{'identifier': {
'scheme': 'UA-ED',
'id': CODES[2],
"name": "company_name"}}]}]}}))]
data = Data(self.tender_id, self.award_ids[0], CODES[0], "company_name",
{"meta": {"sourceRequests": [self.request_ids[0]]}})
self.assertEqual(self.edrpou_codes_queue.get(), data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0])])
def test_upload_to_doc_service_queue_loop_exit(self, mrequest,
gevent_sleep):
""" Test LoopExit for upload_to_doc_service_queue """
gevent_sleep.side_effect = custom_sleep
self.process_tracker.set_item(self.tender_id, self.award_id, 2)
self.worker.upload_to_doc_service_queue = MagicMock()
self.worker.upload_to_doc_service_queue.peek.side_effect = generate_answers(
answers=[LoopExit(), self.data, self.data], default=LoopExit())
mrequest.post(self.url, [{
'json': self.stat_200(),
'status_code': 200
} for _ in range(2)])
self.worker.start()
sleep(1)
self.assertEqual(self.upload_to_tender_queue.get(), self.tender_data)
self.assertEqual(self.upload_to_tender_queue.get(), self.tender_data)
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
self.assertIsNotNone(
mrequest.request_history[1].headers['X-Client-Request-ID'])
self.assertEqual(self.process_tracker.processing_items,
{item_key(self.tender_id, self.award_id): 2})
def test_process_412(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
api_server_bottle = Bottle()
api_server = WSGIServer(('127.0.0.1', 20604),
api_server_bottle,
log=None)
setup_routing(api_server_bottle, response_spore)
setup_routing(api_server_bottle,
response_412,
path='/api/2.3/tenders/{}/awards/{}/documents'.format(
self.tender_id, self.award_id),
method='POST')
api_server.start()
self.worker.client = TendersClient('',
host_url='http://127.0.0.1:20604',
api_version='2.3')
setup_routing(api_server_bottle,
generate_response,
path='/api/2.3/tenders/{}/awards/{}/documents'.format(
self.tender_id, self.award_id),
method='POST')
self.assertEqual(self.worker.client.headers['Cookie'],
'SERVER_ID={}'.format(SPORE_COOKIES))
self.upload_to_tender_queue.put(self.data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
self.assertEqual(self.upload_to_tender_queue.qsize(), 1)
self.shutdown_when_done(self.worker)
self.assertEqual(self.worker.client.headers['Cookie'],
'SERVER_ID={}'.format(COOKIES_412))
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.worker.retry_upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertItemsEqual(self.process_tracker.processing_items.keys(), [])
api_server.stop()
