def setUp(self):
self.tender_id = uuid.uuid4().hex
self.award_id = uuid.uuid4().hex
self.qualification_id = uuid.uuid4().hex
self.document_id = generate_doc_id()
self.process_tracker = ProcessTracker(db=MagicMock())
self.process_tracker.set_item(self.tender_id, self.award_id, 1)
self.upload_to_doc_service_queue = Queue(10)
self.upload_to_tender_queue = Queue(10)
self.sleep_change_value = APIRateController()
self.sna = event.Event()
self.sna.set()
self.data = Data(self.tender_id, self.award_id, '123', 'awards', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.qualification_data = Data(self.tender_id, self.qualification_id,
'123', 'qualifications', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.doc_service_client = DocServiceClient(host='127.0.0.1',
port='80',
user='',
password='')
self.worker = UploadFileToDocService(self.upload_to_doc_service_queue,
self.upload_to_tender_queue,
self.process_tracker,
self.doc_service_client, self.sna,
self.sleep_change_value)
self.url = '{url}'.format(url=self.doc_service_client.url)
def setUp(self):
self.tender_id = uuid.uuid4().hex
self.award_id = uuid.uuid4().hex
self.qualification_id = uuid.uuid4().hex
self.document_id = generate_doc_id()
self.process_tracker = ProcessTracker(db=MagicMock())
self.process_tracker.set_item(self.tender_id, self.award_id, 1)
self.upload_to_tender_queue = Queue(10)
self.url = 'http://127.0.0.1:20604'
self.sleep_change_value = APIRateController()
self.data = Data(self.tender_id, self.award_id, '123', 'awards', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.qualification_data = Data(self.tender_id, self.qualification_id,
'123', 'qualifications', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.client = MagicMock()
self.worker = UploadFileToTender(self.client,
self.upload_to_tender_queue,
self.process_tracker, MagicMock(),
self.sleep_change_value)
self.worker.retry_upload_to_tender_queue = Queue(10)
def test_unprocessed_items(self):
data = Data('1', '2', '123', 'awards', {
'meta': {
'id': '333'
},
'test_data': 'test_data'
})
self.redis.set("unprocessed_{}".format(data.doc_id()), dumps(data))
self.worker = EdrDataBridge(config)
self.assertEqual(self.worker.upload_to_doc_service_queue.get(), data)
def test_scanner_to_upload_to_doc_service(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.worker = EdrDataBridge(config)
setup_routing(self.api_server_bottle,
get_tender_response,
path='/api/2.3/tenders/123')
setup_routing(self.api_server_bottle,
get_tenders_response,
path='/api/2.3/tenders')
setup_routing(self.proxy_server_bottle,
get_proxy_response,
path='/api/1.0/verify')
setup_routing(self.doc_server_bottle,
get_doc_service_response,
path='/upload',
method='POST')
self.worker.scanner()
self.worker.filter_tender()
self.worker.edr_handler()
self.worker.upload_file_to_doc_service()
self.sleep_until_done(self.worker, self.is_working_all)
data = Data(
'123', qualification_ids[2], CODES[2], "qualifications", {
'meta': {},
'url':
'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f',
'format': 'application/yaml',
'hash': 'md5:9a0364b9e99bb480dd25e1f0284c8555',
'title': file_name
})
self.assertEqual(self.worker.edrpou_codes_queue.qsize(), 0)
self.assertEqual(self.worker.upload_to_doc_service_queue.qsize(), 0)
self.check_data_objects(self.worker.upload_to_tender_queue.get(), data)
self.assertEqual(self.worker.filtered_tender_ids_queue.qsize(), 0)
def test_scanner_to_edr_handler(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.worker = EdrDataBridge(config)
setup_routing(self.api_server_bottle,
get_tender_response,
path='/api/2.3/tenders/123')
setup_routing(self.api_server_bottle,
get_tenders_response,
path='/api/2.3/tenders')
setup_routing(self.proxy_server_bottle,
get_proxy_response,
path='/api/1.0/verify')
self.worker.scanner()
self.worker.filter_tender()
self.worker.edr_handler()
self.sleep_until_done(self.worker, self.is_working_all)
data = Data(
'123', qualification_ids[2], CODES[2], "qualifications",
{'meta': {
'sourceRequests': [request_ids[0], request_ids[0]]
}})
self.check_data_objects(self.worker.upload_to_doc_service_queue.get(),
data)
self.assertEqual(self.worker.edrpou_codes_queue.qsize(), 0)
self.assertEqual(self.worker.filtered_tender_ids_queue.qsize(), 0)
def test_worker_award_with_cancelled_lot(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.client.request.return_value = ResponseMock({'X-Request-ID': self.request_ids[0]},
munchify({'prev_page': {'offset': '123'},
'next_page': {'offset': '1234'},
'data': {'status': "active.pre-qualification",
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'lots': [{'status': 'cancelled',
'id': '123456789'},
{'status': 'active',
'id': '12345678'}],
'awards': [{'id': self.award_ids[0],
'bid_id': self.bid_ids[0],
'status': 'pending',
'suppliers': [{'identifier': {
'scheme': 'UA-EDR',
'id': CODES[0]}}],
'lotID': '123456789'},
{'id': self.award_ids[1],
'bid_id': self.bid_ids[1],
'status': 'pending',
'suppliers': [{'identifier': {
'scheme': 'UA-EDR',
'id': CODES[1]}}],
'lotID': '12345678'}]}}))
data = Data(self.tender_id, self.award_ids[1], CODES[1], 'awards',
{'meta': {'sourceRequests': [self.request_ids[0]]}})
for edrpou in [data]:
self.check_data_objects(self.edrpou_codes_queue.get(), edrpou)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[1])])
def test_worker_award(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.client.request.side_effect = [ResponseMock({'X-Request-ID': self.request_ids[0]},
munchify({'prev_page': {'offset': '123'},
'next_page': {'offset': '1234'},
'data': {'status': "active.pre-qualification",
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'awards': [
self.awards(0, 0, 'pending', CODES[0]),
self.awards(1, 1, 'pending', CODES[1]),
self.awards(2, 2, 'pending', CODES[2]),
self.awards(3, 3, 'unsuccessful',
CODES[2]),
{'id': self.bid_ids[4],
'tenderers': [{'identifier': {
'scheme': 'UA-ED',
'id': CODES[2]}}]}]}}))]
for i in range(3):
data = Data(self.tender_id, self.award_ids[i], CODES[i], 'awards',
{'meta': {'sourceRequests': [self.request_ids[0]]}})
self.check_data_objects(self.edrpou_codes_queue.get(), data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0]), item_key(self.tender_id, self.award_ids[1]),
item_key(self.tender_id, self.award_ids[2])])
def test_filtered_tender_ids_queue_loop_exit(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
filtered_tender_ids_queue = MagicMock()
filtered_tender_ids_queue.peek.side_effect = [LoopExit(), self.tender_id]
self.client.request.return_value = self.response
first_data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards',
{'meta': {'sourceRequests': [self.request_ids[0]]}})
self.worker.filtered_tender_ids_queue = filtered_tender_ids_queue
self.check_data_objects(self.edrpou_codes_queue.get(), first_data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0])])
def datum(self):
return Data(
self.tender_id, self.award_id, '123', 'awards', {
u'meta': {
u'id': self.document_id
},
u'url':
u'http://docs-sandbox.openprocurement.org/get/8ccbfde0c6804143b119d9168452cb6f',
u'format': u'application/yaml',
u'hash': u'md5:9a0364b9e99bb480dd25e1f0284c8555',
u'title': file_name
})
def test_get_tender_exception(self, gevent_sleep):
""" We must not lose tender after restart filter worker """
gevent_sleep.side_effect = custom_sleep
self.client.request.side_effect = [Exception(), self.response]
data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards',
{'meta': {'sourceRequests': [self.request_ids[0]]}})
self.check_data_objects(self.edrpou_codes_queue.get(), data)
self.assertEqual(self.worker.sleep_change_value.time_between_requests, 0)
gevent_sleep.assert_called_with_once(1)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0])])
self.assertEqual(self.edrpou_codes_queue.qsize(), 0)
def temp_important_part_for_item(self, response, tender, item, item_name,
code):
self.process_tracker.set_item(tender['id'], item['id'])
document_id = generate_doc_id()
tender_data = Data(
tender['id'], item['id'], str(code), item_name + "s", {
'meta': {
'id': document_id,
'author': author,
'sourceRequests': [response.headers['X-Request-ID']]
}
})
self.edrpou_codes_queue.put(tender_data)
def test_worker_dead(self, gevent_sleep):
""" Test that worker will process tender after exception """
gevent_sleep.side_effect = custom_sleep
self.filtered_tender_ids_queue.put(self.tender_id)
self.client.request.side_effect = [
ResponseMock({'X-Request-ID': self.request_ids[i]},
munchify({'prev_page': {'offset': '123'},
'next_page': {'offset': '1234'},
'data': {
'status': "active.pre-qualification",
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'awards': [self.awards(i, i, 'pending', CODES[0])]}})) for i in range(2)]
for i in range(2):
data = Data(self.tender_id, self.award_ids[i], CODES[0], 'awards',
{'meta': {'sourceRequests': [self.request_ids[i]]}})
self.check_data_objects(self.edrpou_codes_queue.get(), data)
self.worker.immortal_jobs['prepare_data'].kill(timeout=1)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[i]) for i in range(2)])
def test_worker_restart(self, gevent_sleep):
""" Process tender after catch Unauthorized exception """
gevent_sleep.side_effect = custom_sleep
self.client.request.side_effect = [
Unauthorized(http_code=403),
Unauthorized(http_code=403),
Unauthorized(http_code=403),
ResponseMock({'X-Request-ID': self.request_ids[0]},
munchify({'prev_page': {'offset': '123'},
'next_page': {'offset': '1234'},
'data': {'status': "active.pre-qualification",
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'awards': [self.awards(0, 0, 'pending', CODES[0]),
self.awards(1, 1, 'unsuccessful', CODES[2])]}}))]
data = Data(self.tender_id, self.award_ids[0], CODES[0], 'awards',
{'meta': {'sourceRequests': [self.request_ids[0]]}})
self.check_data_objects(self.edrpou_codes_queue.get(), data)
self.assertItemsEqual(self.process_tracker.processing_items.keys(),
[item_key(self.tender_id, self.award_ids[0])])
def test_412(self, gevent_sleep):
gevent_sleep.side_effect = custom_sleep
self.worker.kill()
filtered_tender_ids_queue = Queue(10)
filtered_tender_ids_queue.put('123')
setup_routing(self.api_server_bottle, response_spore)
setup_routing(self.api_server_bottle, generate_response, path='/api/2.3/tenders/123')
client = TendersClientSync('', host_url='http://127.0.0.1:20604', api_version='2.3')
self.assertEqual(client.headers['Cookie'],
'SERVER_ID={}'.format(SPORE_COOKIES)) # check that response_spore set cookies
worker = FilterTenders.spawn(client, filtered_tender_ids_queue, self.edrpou_codes_queue, self.process_tracker,
MagicMock(), self.sleep_change_value)
data = Data('123', '124', CODES[0], 'awards', {'meta': {'sourceRequests': ['125']}})
for i in [data]:
self.check_data_objects(self.edrpou_codes_queue.get(), i)
self.assertEqual(client.headers['Cookie'],
'SERVER_ID={}'.format(COOKIES_412)) # check that response_412 change cookies
self.assertEqual(self.edrpou_codes_queue.qsize(), 0)
self.assertItemsEqual(self.process_tracker.processing_items.keys(), ['123_124'])
worker.shutdown()
del worker