def test_worker_award(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.client.request.side_effect = [ResponseMock({'X-Request-ID': self.request_ids[0]},
munchify({'prev_page': {'offset': '123'},
'next_page': {'offset': '1234'},
'data': {'status': "active.pre-qualification",
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'awards': [
self.awards(0, 0, 'pending', CODES[0]),
self.awards(1, 1, 'pending', CODES[1]),
self.awards(2, 2, 'pending', CODES[2]),
self.awards(3, 3, 'unsuccessful',
CODES[2]),
{'id': self.bid_ids[4],
'tenderers': [{'identifier': {
'scheme': 'UA-ED',
'id': CODES[2]}}]}]}}))]
for i in range(3):
data = Data(self.tender_id, self.award_ids[i], CODES[i], 'awards',
{'meta': {'sourceRequests': [self.request_ids[0]]}})
self.check_data_objects(self.edrpou_codes_queue.get(), data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0]), item_key(self.tender_id, self.award_ids[1]),
item_key(self.tender_id, self.award_ids[2])])
def test_update_processing_items(self):
self.process_tracker.processing_items = {
item_key(self.tender_id, self.item_id): 2
}
self.assertEqual(self.process_tracker.processing_items,
{item_key(self.tender_id, self.item_id): 2})
self.process_tracker.update_processing_items(self.tender_id,
self.item_id)
self.assertEqual(self.process_tracker.processing_items,
{item_key(self.tender_id, self.item_id): 1})
self.process_tracker.update_processing_items(self.tender_id,
self.item_id)
self.assertEqual(self.process_tracker.processing_items, {})
def test_filtered_tender_ids_queue_loop_exit(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
filtered_tender_ids_queue = MagicMock()
filtered_tender_ids_queue.peek.side_effect = [
LoopExit(), self.tender_id
]
self.client.request.return_value = ResponseMock(
{'X-Request-ID': self.request_ids[0]},
munchify({
'prev_page': {
'offset': '123'
},
'next_page': {
'offset': '1234'
},
'data': {
'status': "active.pre-qualification",
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'awards': [self.awards(0, 0, 'pending', CODES[0])]
}
}))
first_data = Data(self.tender_id, self.award_ids[0], CODES[0],
'awards',
{'meta': {
'sourceRequests': [self.request_ids[0]]
}})
self.worker.filtered_tender_ids_queue = filtered_tender_ids_queue
self.check_data_objects(self.edrpou_codes_queue.get(), first_data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0])])
def test_retry_upload_to_tender(self, mrequest, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
mrequest.post(self.url, json=self.stat_200(), status_code=200)
self.client._create_tender_resource_item.side_effect = [
Unauthorized(http_code=401),
Unauthorized(http_code=403),
Unauthorized(http_code=429),
self.get_tender()
]
self.upload_to_doc_service_queue.put(self.data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(mrequest.call_count, 1)
self.assertEqual(mrequest.request_history[0].url,
u'127.0.0.1:80/upload')
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
self.assertEqual(self.process_tracker.processing_items,
{}) # test that item removed from processing_items
self.assertEqual(self.client._create_tender_resource_item.call_count,
4) # check upload to tender
def test_retry_doc_service(self, mrequest, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
doc_service_client = DocServiceClient(host='127.0.0.1',
port='80',
user='',
password='')
mrequest.post(self.url, [{
'text': '',
'status_code': 401
} for _ in range(6)] + [{
'json': {
'data': {
'url': 'test url',
'format': 'application/yaml',
'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555',
'title': file_name
}
},
'status_code': 200
}])
self.upload_to_doc_service_queue.put(self.data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.upload_to_tender_queue.qsize(), 1,
'Queue should be have 1 element')
self.assertEqual(mrequest.call_count, 7)
self.assertEqual(mrequest.request_history[0].url,
u'127.0.0.1:80/upload')
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
def test_get_tender_429(self, gevent_sleep):
""" We must not lose tender after restart filter worker """
gevent_sleep.side_effect = custom_sleep
self.client.request.side_effect = [
ResourceError(http_code=429),
ResponseMock({'X-Request-ID': self.request_ids[0]},
munchify({
'prev_page': {
'offset': '123'
},
'next_page': {
'offset': '1234'
},
'data': {
'status': "active.pre-qualification",
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'awards':
[self.awards(0, 0, 'pending', CODES[0])]
}
}))
]
data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards',
{'meta': {
'sourceRequests': [self.request_ids[0]]
}})
self.sleep_change_value.increment_step = 2
self.sleep_change_value.decrement_step = 1
self.check_data_objects(self.edrpou_codes_queue.get(), data)
self.assertEqual(self.worker.sleep_change_value.time_between_requests,
1)
gevent_sleep.assert_called_with_once(1)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0])])
self.assertEqual(self.edrpou_codes_queue.qsize(), 0)
def test_worker_dead(self, gevent_sleep):
""" Test that worker will process tender after exception """
gevent_sleep.side_effect = custom_sleep
self.filtered_tender_ids_queue.put(self.tender_id)
self.client.request.side_effect = [
ResponseMock({'X-Request-ID': self.request_ids[i]},
munchify({
'prev_page': {
'offset': '123'
},
'next_page': {
'offset': '1234'
},
'data': {
'status': "active.pre-qualification",
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'awards':
[self.awards(i, i, 'pending', CODES[0])]
}
})) for i in range(2)
]
for i in range(2):
data = Data(self.tender_id, self.award_ids[i], CODES[0], 'awards',
{'meta': {
'sourceRequests': [self.request_ids[i]]
}})
self.check_data_objects(self.edrpou_codes_queue.get(), data)
self.worker.job.kill(timeout=1)
self.assertItemsEqual(
self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[i]) for i in range(2)])
def test_worker_award_with_cancelled_lot(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.client.request.return_value = ResponseMock({'X-Request-ID': self.request_ids[0]},
munchify({'prev_page': {'offset': '123'},
'next_page': {'offset': '1234'},
'data': {'status': "active.pre-qualification",
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'lots': [{'status': 'cancelled',
'id': '123456789'},
{'status': 'active',
'id': '12345678'}],
'awards': [{'id': self.award_ids[0],
'bid_id': self.bid_ids[0],
'status': 'pending',
'suppliers': [{'identifier': {
'scheme': 'UA-EDR',
'id': CODES[0]}}],
'lotID': '123456789'},
{'id': self.award_ids[1],
'bid_id': self.bid_ids[1],
'status': 'pending',
'suppliers': [{'identifier': {
'scheme': 'UA-EDR',
'id': CODES[1]}}],
'lotID': '12345678'}]}}))
data = Data(self.tender_id, self.award_ids[1], CODES[1], 'awards',
{'meta': {'sourceRequests': [self.request_ids[0]]}})
for edrpou in [data]:
self.check_data_objects(self.edrpou_codes_queue.get(), edrpou)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[1])])
def test_successfully_uploaded_to_tender_retry(self):
self.worker.retry_upload_to_tender_queue.put(self.data)
self.assertEqual(self.worker.process_tracker.processing_items,
{item_key(self.tender_id, self.award_id): 1})
self.worker.successfully_uploaded_to_tender(self.data, True)
self.assertEqual(self.worker.retry_upload_to_tender_queue.qsize(), 0)
self.assertEqual(self.worker.process_tracker.processing_items, {})
def test_set_item(self):
self.assertEqual(self.process_tracker.processing_items, {})
self.assertEqual(self.process_tracker.tender_documents_to_process, {})
self.process_tracker.set_item(self.tender_id, self.item_id, 1)
self.assertEqual(self.process_tracker.processing_items,
{item_key(self.tender_id, self.item_id): 1})
self.assertEqual(self.process_tracker.tender_documents_to_process,
{db_key(self.tender_id): 1})
def _update_processing_items(self, tender_id, item_id, document_id):
key = item_key(tender_id, item_id)
if self.processing_items[key] > 1:
self.processing_items[key] -= 1
else:
self.processed_items[key] = datetime.now()
self._remove_unprocessed_item(document_id)
del self.processing_items[key]
def test_successful_upload(self, mrequest, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
mrequest.post(self.url, json=self.stat_200(), status_code=200)
self.upload_to_doc_service_queue.put(self.data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.upload_to_tender_queue.qsize(), 1,
'Queue should be have 1 element')
self.assertEqual(mrequest.call_count, 1)
self.assertEqual(mrequest.request_history[0].url,
u'127.0.0.1:80/upload')
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
def test_filtered_tender_ids_queue_loop_exit(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
filtered_tender_ids_queue = MagicMock()
filtered_tender_ids_queue.peek.side_effect = [LoopExit(), self.tender_id]
self.client.request.return_value = self.response
first_data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards',
{'meta': {'sourceRequests': [self.request_ids[0]]}})
self.worker.filtered_tender_ids_queue = filtered_tender_ids_queue
self.check_data_objects(self.edrpou_codes_queue.get(), first_data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0])])
def test_get_tender_exception(self, gevent_sleep):
""" We must not lose tender after restart filter worker """
gevent_sleep.side_effect = custom_sleep
self.client.request.side_effect = [Exception(), self.response]
data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards',
{'meta': {'sourceRequests': [self.request_ids[0]]}})
self.check_data_objects(self.edrpou_codes_queue.get(), data)
self.assertEqual(self.worker.sleep_change_value.time_between_requests, 0)
gevent_sleep.assert_called_with_once(1)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0])])
self.assertEqual(self.edrpou_codes_queue.qsize(), 0)
def test_request_failed_item_status_change(self, mrequest, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
mrequest.post(self.url, json=self.stat_200(), status_code=200)
self.process_tracker.set_item(self.tender_id, self.qualification_id, 1)
self.upload_to_doc_service_queue.put(self.data)
self.upload_to_doc_service_queue.put(self.qualification_data)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.upload_to_tender_queue.get(), self.data)
self.assertEqual(self.upload_to_tender_queue.get(),
self.qualification_data)
self.assertEqual(mrequest.call_count, 2)
self.assertEqual(mrequest.request_history[0].url,
u'127.0.0.1:80/upload')
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
self.assertEqual(
self.process_tracker.processing_items, {
item_key(self.tender_id, self.award_id): 1,
item_key(self.tender_id, self.qualification_id): 1
})
def test_412(self, mrequest, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
mrequest.post(self.url, json=self.stat_200(), status_code=200)
api_server_bottle = Bottle()
api_server = WSGIServer(('127.0.0.1', 20604),
api_server_bottle,
log=None)
setup_routing(api_server_bottle, response_spore)
setup_routing(api_server_bottle,
response_412,
path='/api/2.3/tenders/{}/awards/{}/documents'.format(
self.tender_id, self.award_id),
method='POST')
api_server.start()
self.worker.client = TendersClientSync(
'', host_url='http://127.0.0.1:20604', api_version='2.3')
setup_routing(api_server_bottle,
generate_response,
path='/api/2.3/tenders/{}/awards/{}/documents'.format(
self.tender_id, self.award_id),
method='POST')
self.assertEqual(
self.worker.client.headers['Cookie'], 'SERVER_ID={}'.format(
SPORE_COOKIES)) # check that response_spore set cookies
self.upload_to_doc_service_queue.put(self.data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 1)
self.worker.start()
self.shutdown_when_done(self.worker)
self.assertEqual(
self.worker.client.headers['Cookie'], 'SERVER_ID={}'.format(
COOKIES_412)) # check that response_412 change cookies
self.assertEqual(self.upload_to_doc_service_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(mrequest.call_count, 1)
self.assertEqual(mrequest.request_history[0].url,
u'127.0.0.1:80/upload')
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
self.assertItemsEqual(self.process_tracker.processing_items.keys(), [])
def test_retry_upload_to_tender(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.client._create_tender_resource_item.side_effect = [
Unauthorized(http_code=401),
Unauthorized(http_code=403),
Unauthorized(http_code=429),
self.get_tender()
]
self.upload_to_tender_queue.put(self.data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
self.assertEqual(self.upload_to_tender_queue.qsize(), 1)
self.shutdown_when_done(self.worker)
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.process_tracker.processing_items,
{}) # test that item removed from processing_items
self.assertEqual(self.client._create_tender_resource_item.call_count,
4) # check upload to tender
def test_worker_restart(self, gevent_sleep):
""" Process tender after catch Unauthorized exception """
gevent_sleep.side_effect = custom_sleep
self.client.request.side_effect = [
Unauthorized(http_code=403),
Unauthorized(http_code=403),
Unauthorized(http_code=403),
ResponseMock({'X-Request-ID': self.request_ids[0]},
munchify({'prev_page': {'offset': '123'},
'next_page': {'offset': '1234'},
'data': {'status': "active.pre-qualification",
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'awards': [self.awards(0, 0, 'pending', CODES[0]),
self.awards(1, 1, 'unsuccessful', CODES[2])]}}))]
data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards',
{'meta': {'sourceRequests': [self.request_ids[0]]}})
self.check_data_objects(self.edrpou_codes_queue.get(), data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0])])
def test_upload_to_doc_service_queue_loop_exit(self, mrequest,
gevent_sleep):
""" Test LoopExit for upload_to_doc_service_queue """
gevent_sleep.side_effect = custom_sleep
self.process_tracker.set_item(self.tender_id, self.award_id, 2)
self.worker.upload_to_doc_service_queue = MagicMock()
self.worker.upload_to_doc_service_queue.peek.side_effect = generate_answers(
answers=[LoopExit(), self.data, self.data], default=LoopExit())
mrequest.post(self.url, [{
'json': self.stat_200(),
'status_code': 200
} for _ in range(2)])
self.worker.start()
sleep(1)
self.assertEqual(self.upload_to_tender_queue.get(), self.data)
self.assertIsNotNone(
mrequest.request_history[0].headers['X-Client-Request-ID'])
self.assertIsNotNone(
mrequest.request_history[1].headers['X-Client-Request-ID'])
self.assertEqual(self.process_tracker.processing_items,
{item_key(self.tender_id, self.award_id): 2})
def test_process_412(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
api_server_bottle = Bottle()
api_server = WSGIServer(('127.0.0.1', 20604),
api_server_bottle,
log=None)
setup_routing(api_server_bottle, response_spore)
setup_routing(api_server_bottle,
response_412,
path='/api/2.3/tenders/{}/awards/{}/documents'.format(
self.tender_id, self.award_id),
method='POST')
api_server.start()
self.worker.client = TendersClient('',
host_url='http://127.0.0.1:20604',
api_version='2.3')
setup_routing(api_server_bottle,
generate_response,
path='/api/2.3/tenders/{}/awards/{}/documents'.format(
self.tender_id, self.award_id),
method='POST')
self.assertEqual(self.worker.client.headers['Cookie'],
'SERVER_ID={}'.format(SPORE_COOKIES))
self.upload_to_tender_queue.put(self.data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_id)])
self.assertEqual(self.upload_to_tender_queue.qsize(), 1)
self.shutdown_when_done(self.worker)
self.assertEqual(self.worker.client.headers['Cookie'],
'SERVER_ID={}'.format(COOKIES_412))
self.assertEqual(self.upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertEqual(self.worker.retry_upload_to_tender_queue.qsize(), 0,
'Queue should be empty')
self.assertItemsEqual(self.process_tracker.processing_items.keys(), [])
api_server.stop()
def set_item(self, tender_id, item_id, docs_amount=0):
self.processing_items[item_key(tender_id, item_id)] = docs_amount
self._add_docs_amount_to_tender(tender_id, docs_amount)
def check_processed_item(self, tender_id, item_id):
"""Check if current tender_id, item_id was already processed"""
return item_key(tender_id, item_id) in self.processed_items.keys()
def check_processing_item(self, tender_id, item_id):
"""Check if current tender_id, item_id is processing"""
return item_key(tender_id, item_id) in self.processing_items.keys()
