def setUp(self):
self.filtered_tender_ids_queue = Queue(10)
self.edrpou_codes_queue = Queue(10)
self.process_tracker = ProcessTracker(Db(config))
self.tender_id = uuid.uuid4().hex
self.sleep_change_value = APIRateController()
self.worker = EdrDataBridge(config)
def setUp(self):
self.tender_id = uuid.uuid4().hex
self.award_id = uuid.uuid4().hex
self.qualification_id = uuid.uuid4().hex
self.document_id = generate_doc_id()
self.process_tracker = ProcessTracker(db=MagicMock())
self.process_tracker.set_item(self.tender_id, self.award_id, 1)
self.upload_to_tender_queue = Queue(10)
self.url = 'http://127.0.0.1:20604'
self.sleep_change_value = APIRateController()
self.data = Data(self.tender_id, self.award_id, '123', 'awards', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.qualification_data = Data(self.tender_id, self.qualification_id,
'123', 'qualifications', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.client = MagicMock()
self.worker = UploadFileToTender(self.client,
self.upload_to_tender_queue,
self.process_tracker, MagicMock(),
self.sleep_change_value)
self.worker.retry_upload_to_tender_queue = Queue(10)
def setUp(self):
self.tender_id = uuid.uuid4().hex
self.award_id = uuid.uuid4().hex
self.qualification_id = uuid.uuid4().hex
self.document_id = generate_doc_id()
self.process_tracker = ProcessTracker(db=MagicMock())
self.process_tracker.set_item(self.tender_id, self.award_id, 1)
self.upload_to_doc_service_queue = Queue(10)
self.upload_to_tender_queue = Queue(10)
self.sleep_change_value = APIRateController()
self.sna = event.Event()
self.sna.set()
self.data = Data(self.tender_id, self.award_id, '123', 'awards', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.qualification_data = Data(self.tender_id, self.qualification_id,
'123', 'qualifications', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.doc_service_client = DocServiceClient(host='127.0.0.1',
port='80',
user='',
password='')
self.worker = UploadFileToDocService(self.upload_to_doc_service_queue,
self.upload_to_tender_queue,
self.process_tracker,
self.doc_service_client, self.sna,
self.sleep_change_value)
self.url = '{url}'.format(url=self.doc_service_client.url)
def setUp(self):
self.process_tracker = ProcessTracker(
MagicMock(has=MagicMock(return_value=False)))
self.tenders_id = [uuid.uuid4().hex for _ in range(4)]
self.sleep_change_value = APIRateController()
self.client = MagicMock()
self.tender_queue = Queue(10)
self.sna = event.Event()
self.sna.set()
self.worker = Scanner.spawn(self.client, self.tender_queue, self.sna,
self.process_tracker,
self.sleep_change_value)
def setUp(self):
self.filtered_tender_ids_queue = Queue(10)
self.edrpou_codes_queue = Queue(10)
self.db = Db(config)
self.process_tracker = ProcessTracker(self.db)
self.tender_id = uuid.uuid4().hex
self.filtered_tender_ids_queue.put(self.tender_id)
self.sleep_change_value = APIRateController()
self.client = MagicMock()
self.sna = event.Event()
self.sna.set()
self.worker = FilterTenders.spawn(self.client, self.filtered_tender_ids_queue, self.edrpou_codes_queue,
self.process_tracker, self.sna, self.sleep_change_value)
self.bid_ids = [uuid.uuid4().hex for _ in range(5)]
self.qualification_ids = [uuid.uuid4().hex for _ in range(5)]
self.award_ids = [uuid.uuid4().hex for _ in range(5)]
self.request_ids = [generate_request_id() for _ in range(2)]
self.response = ResponseMock({'X-Request-ID': self.request_ids[0]},
munchify({'prev_page': {'offset': '123'},
'next_page': {'offset': '1234'},
'data': {'status': "active.pre-qualification",
'id': self.tender_id,
'procurementMethodType': 'aboveThresholdEU',
'awards': [self.awards(0, 0, 'pending', CODES[0])]}}))
def __init__(self, config):
super(EdrDataBridge, self).__init__()
self.config = config
api_server = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
ro_api_server = self.config_get(
'public_tenders_api_server') or api_server
buffers_size = self.config_get('buffers_size') or 500
self.delay = self.config_get('delay') or 15
self.increment_step = self.config_get('increment_step') or 1
self.decrement_step = self.config_get('decrement_step') or 1
self.sleep_change_value = APIRateController(self.increment_step,
self.decrement_step)
self.doc_service_host = self.config_get('doc_service_server')
self.doc_service_port = self.config_get('doc_service_port') or 6555
self.sandbox_mode = os.environ.get('SANDBOX_MODE', 'False')
self.time_to_live = self.config_get('time_to_live') or 300
# init clients
self.tenders_sync_client = TendersClientSync(
'', host_url=ro_api_server, api_version=self.api_version)
self.client = TendersClient(self.config_get('api_token'),
host_url=api_server,
api_version=self.api_version)
self.proxy_client = ProxyClient(
host=self.config_get('proxy_server'),
user=self.config_get('proxy_user'),
password=self.config_get('proxy_password'),
port=self.config_get('proxy_port'),
version=self.config_get('proxy_version'))
self.doc_service_client = DocServiceClient(
host=self.doc_service_host,
port=self.doc_service_port,
user=self.config_get('doc_service_user'),
password=self.config_get('doc_service_password'))
# init queues for workers
self.filtered_tender_ids_queue = Queue(
maxsize=buffers_size
) # queue of tender IDs with appropriate status
self.edrpou_codes_queue = Queue(
maxsize=buffers_size
) # queue with edrpou codes (Data objects stored in it)
self.upload_to_doc_service_queue = Queue(
maxsize=buffers_size
) # queue with info from EDR (Data.file_content)
self.upload_to_tender_queue = Queue(maxsize=buffers_size)
# blockers
self.initialization_event = event.Event()
self.services_not_available = event.Event()
self.services_not_available.set()
self.db = Db(config)
self.process_tracker = ProcessTracker(self.db, self.time_to_live)
unprocessed_items = self.process_tracker.get_unprocessed_items()
for item in unprocessed_items:
self.upload_to_doc_service_queue.put(loads(item))
# Workers
self.scanner = partial(
Scanner.spawn,
tenders_sync_client=self.tenders_sync_client,
filtered_tender_ids_queue=self.filtered_tender_ids_queue,
services_not_available=self.services_not_available,
process_tracker=self.process_tracker,
sleep_change_value=self.sleep_change_value,
delay=self.delay)
self.filter_tender = partial(
FilterTenders.spawn,
tenders_sync_client=self.tenders_sync_client,
filtered_tender_ids_queue=self.filtered_tender_ids_queue,
edrpou_codes_queue=self.edrpou_codes_queue,
process_tracker=self.process_tracker,
services_not_available=self.services_not_available,
sleep_change_value=self.sleep_change_value,
delay=self.delay)
self.edr_handler = partial(
EdrHandler.spawn,
proxy_client=self.proxy_client,
edrpou_codes_queue=self.edrpou_codes_queue,
upload_to_doc_service_queue=self.upload_to_doc_service_queue,
process_tracker=self.process_tracker,
services_not_available=self.services_not_available,
delay=self.delay)
self.upload_file_to_doc_service = partial(
UploadFileToDocService.spawn,
upload_to_doc_service_queue=self.upload_to_doc_service_queue,
upload_to_tender_queue=self.upload_to_tender_queue,
process_tracker=self.process_tracker,
doc_service_client=self.doc_service_client,
services_not_available=self.services_not_available,
sleep_change_value=self.sleep_change_value,
delay=self.delay)
self.upload_file_to_tender = partial(
UploadFileToTender.spawn,
client=self.client,
upload_to_tender_queue=self.upload_to_tender_queue,
process_tracker=self.process_tracker,
services_not_available=self.services_not_available,
sleep_change_value=self.sleep_change_value,
delay=self.delay)
def setUp(self):
self.process_tracker = ProcessTracker(self.db)
self.tender_id = "111"
self.item_id = "222"
self.document_id = "333"