def setUp(self): self.filtered_tender_ids_queue = Queue(10) self.edrpou_codes_queue = Queue(10) self.process_tracker = ProcessTracker(Db(config)) self.tender_id = uuid.uuid4().hex self.sleep_change_value = APIRateController() self.worker = EdrDataBridge(config)
def setUp(self): self.tender_id = uuid.uuid4().hex self.award_id = uuid.uuid4().hex self.qualification_id = uuid.uuid4().hex self.document_id = generate_doc_id() self.process_tracker = ProcessTracker(db=MagicMock()) self.process_tracker.set_item(self.tender_id, self.award_id, 1) self.upload_to_tender_queue = Queue(10) self.url = 'http://127.0.0.1:20604' self.sleep_change_value = APIRateController() self.data = Data(self.tender_id, self.award_id, '123', 'awards', { 'meta': { 'id': self.document_id }, 'test_data': 'test_data' }) self.qualification_data = Data(self.tender_id, self.qualification_id, '123', 'qualifications', { 'meta': { 'id': self.document_id }, 'test_data': 'test_data' }) self.client = MagicMock() self.worker = UploadFileToTender(self.client, self.upload_to_tender_queue, self.process_tracker, MagicMock(), self.sleep_change_value) self.worker.retry_upload_to_tender_queue = Queue(10)
def setUp(self): self.tender_id = uuid.uuid4().hex self.award_id = uuid.uuid4().hex self.qualification_id = uuid.uuid4().hex self.document_id = generate_doc_id() self.process_tracker = ProcessTracker(db=MagicMock()) self.process_tracker.set_item(self.tender_id, self.award_id, 1) self.upload_to_doc_service_queue = Queue(10) self.upload_to_tender_queue = Queue(10) self.sleep_change_value = APIRateController() self.sna = event.Event() self.sna.set() self.data = Data(self.tender_id, self.award_id, '123', 'awards', { 'meta': { 'id': self.document_id }, 'test_data': 'test_data' }) self.qualification_data = Data(self.tender_id, self.qualification_id, '123', 'qualifications', { 'meta': { 'id': self.document_id }, 'test_data': 'test_data' }) self.doc_service_client = DocServiceClient(host='127.0.0.1', port='80', user='', password='') self.worker = UploadFileToDocService(self.upload_to_doc_service_queue, self.upload_to_tender_queue, self.process_tracker, self.doc_service_client, self.sna, self.sleep_change_value) self.url = '{url}'.format(url=self.doc_service_client.url)
def setUp(self): self.process_tracker = ProcessTracker(MagicMock(has=MagicMock(return_value=False))) self.tenders_id = [uuid.uuid4().hex for _ in range(4)] self.sleep_change_value = APIRateController() self.client = MagicMock() self.tender_queue = Queue(10) self.worker = Scanner.spawn(self.client, self.tender_queue, MagicMock(), self.process_tracker, self.sleep_change_value)
def setUp(self): self.filtered_tender_ids_queue = Queue(10) self.edrpou_codes_queue = Queue(10) self.process_tracker = ProcessTracker() self.tender_id = uuid.uuid4().hex self.filtered_tender_ids_queue.put(self.tender_id) self.sleep_change_value = APIRateController() self.client = MagicMock() self.worker = FilterTenders.spawn(self.client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, self.process_tracker, MagicMock(), self.sleep_change_value) self.bid_ids = [uuid.uuid4().hex for _ in range(5)] self.qualification_ids = [uuid.uuid4().hex for _ in range(5)] self.award_ids = [uuid.uuid4().hex for _ in range(5)] self.request_ids = [generate_request_id() for _ in range(2)]
def setUp(self): self.filtered_tender_ids_queue = Queue(10) self.edrpou_codes_queue = Queue(10) self.db = Db(config) self.process_tracker = ProcessTracker(self.db) self.tender_id = uuid.uuid4().hex self.filtered_tender_ids_queue.put(self.tender_id) self.sleep_change_value = APIRateController() self.client = MagicMock() self.sna = event.Event() self.sna.set() self.worker = FilterTenders.spawn(self.client, self.filtered_tender_ids_queue, self.edrpou_codes_queue, self.process_tracker, self.sna, self.sleep_change_value) self.bid_ids = [uuid.uuid4().hex for _ in range(5)] self.qualification_ids = [uuid.uuid4().hex for _ in range(5)] self.award_ids = [uuid.uuid4().hex for _ in range(5)] self.request_ids = [generate_request_id() for _ in range(2)] self.response = ResponseMock({'X-Request-ID': self.request_ids[0]}, munchify({'prev_page': {'offset': '123'}, 'next_page': {'offset': '1234'}, 'data': {'status': "active.pre-qualification", 'id': self.tender_id, 'procurementMethodType': 'aboveThresholdEU', 'awards': [self.awards(0, 0, 'pending', CODES[0])]}}))
def __init__(self, config): super(EdrDataBridge, self).__init__() self.config = config api_server = self.config_get('tenders_api_server') self.api_version = self.config_get('tenders_api_version') ro_api_server = self.config_get( 'public_tenders_api_server') or api_server buffers_size = self.config_get('buffers_size') or 500 self.delay = self.config_get('delay') or 15 self.increment_step = self.config_get('increment_step') or 1 self.decrement_step = self.config_get('decrement_step') or 1 self.sleep_change_value = APIRateController(self.increment_step, self.decrement_step) self.doc_service_host = self.config_get('doc_service_server') self.doc_service_port = self.config_get('doc_service_port') or 6555 self.sandbox_mode = os.environ.get('SANDBOX_MODE', 'False') self.time_to_live = self.config_get('time_to_live') or 300 # init clients self.tenders_sync_client = TendersClientSync( '', host_url=ro_api_server, api_version=self.api_version) self.client = TendersClient(self.config_get('api_token'), host_url=api_server, api_version=self.api_version) self.proxy_client = ProxyClient( host=self.config_get('proxy_server'), user=self.config_get('proxy_user'), password=self.config_get('proxy_password'), port=self.config_get('proxy_port'), version=self.config_get('proxy_version')) self.doc_service_client = DocServiceClient( host=self.doc_service_host, port=self.doc_service_port, user=self.config_get('doc_service_user'), password=self.config_get('doc_service_password')) # init queues for workers self.filtered_tender_ids_queue = Queue( maxsize=buffers_size ) # queue of tender IDs with appropriate status self.edrpou_codes_queue = Queue( maxsize=buffers_size ) # queue with edrpou codes (Data objects stored in it) self.upload_to_doc_service_queue = Queue( maxsize=buffers_size ) # queue with info from EDR (Data.file_content) self.upload_to_tender_queue = Queue(maxsize=buffers_size) # blockers self.initialization_event = event.Event() self.services_not_available = event.Event() self.services_not_available.set() self.db = Db(config) self.process_tracker = ProcessTracker(self.db, self.time_to_live) unprocessed_items = self.process_tracker.get_unprocessed_items() for item in unprocessed_items: self.upload_to_doc_service_queue.put(loads(item)) # Workers self.scanner = partial( Scanner.spawn, tenders_sync_client=self.tenders_sync_client, filtered_tender_ids_queue=self.filtered_tender_ids_queue, services_not_available=self.services_not_available, process_tracker=self.process_tracker, sleep_change_value=self.sleep_change_value, delay=self.delay) self.filter_tender = partial( FilterTenders.spawn, tenders_sync_client=self.tenders_sync_client, filtered_tender_ids_queue=self.filtered_tender_ids_queue, edrpou_codes_queue=self.edrpou_codes_queue, process_tracker=self.process_tracker, services_not_available=self.services_not_available, sleep_change_value=self.sleep_change_value, delay=self.delay) self.edr_handler = partial( EdrHandler.spawn, proxy_client=self.proxy_client, edrpou_codes_queue=self.edrpou_codes_queue, upload_to_doc_service_queue=self.upload_to_doc_service_queue, process_tracker=self.process_tracker, services_not_available=self.services_not_available, delay=self.delay) self.upload_file_to_doc_service = partial( UploadFileToDocService.spawn, upload_to_doc_service_queue=self.upload_to_doc_service_queue, upload_to_tender_queue=self.upload_to_tender_queue, process_tracker=self.process_tracker, doc_service_client=self.doc_service_client, services_not_available=self.services_not_available, sleep_change_value=self.sleep_change_value, delay=self.delay) self.upload_file_to_tender = partial( UploadFileToTender.spawn, client=self.client, upload_to_tender_queue=self.upload_to_tender_queue, process_tracker=self.process_tracker, services_not_available=self.services_not_available, sleep_change_value=self.sleep_change_value, delay=self.delay)