def setUp(self): #self._testMethodName self.app = Bottle() setup_routing(self.app) self.server = WSGIServer(('localhost', 20602), self.app, log=None) self.server.start() self.client = ContractingClient(API_KEY, host_url=HOST_URL, api_version=API_VERSION) with open(ROOT + 'contract_' + TEST_CONTRACT_KEYS.contract_id + '.json') as contract: self.contract = munchify(load(contract)) self.contract.update({'access':{"token": API_KEY}})
def contracting_client_init(self): logger.info('Initialization contracting clients.', extra=journal_context({"MESSAGE_ID": DATABRIDGE_INFO}, {})) self.contracting_client = ContractingClient( self.config_get('api_token'), host_url=self.contracting_api_server, api_version=self.contracting_api_version ) self.contracting_client_ro = self.contracting_client if self.config_get('public_tenders_api_server'): if self.api_server == self.contracting_api_server and self.api_version == self.contracting_api_version: self.contracting_client_ro = ContractingClient( '', host_url=self.ro_api_server, api_version=self.api_version )
def __init__(self, config): super(ContractingDataBridge, self).__init__() self.config = config self.tenders_sync_client = TendersClientSync('', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.client = TendersClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.contracting_client = ContractingClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version') ) self.initial_sync_point = {} self.tenders_queue = Queue(maxsize=500) self.handicap_contracts_queue = Queue(maxsize=500) self.contracts_put_queue = Queue(maxsize=500) self.contracts_retry_put_queue = Queue(maxsize=500)
class ContractingUserTestCase(unittest.TestCase): """""" def setUp(self): #self._testMethodName self.app = Bottle() setup_routing(self.app) self.server = WSGIServer(('localhost', 20602), self.app, log=None) self.server.start() self.client = ContractingClient(API_KEY, host_url=HOST_URL, api_version=API_VERSION) with open(ROOT + 'contract_' + TEST_CONTRACT_KEYS.contract_id + '.json') as contract: self.contract = munchify(load(contract)) self.contract.update({'access': {"token": API_KEY}}) def tearDown(self): self.server.stop() ########################################################################### # CREATE ITEM TEST ########################################################################### def test_create_contract(self): setup_routing(self.app, routs=["contract_create"]) contract = munchify({'data': 'contract'}) self.client.create_contract(contract) ########################################################################### # DOCUMENTS FILE TEST ########################################################################### def test_upload_contract_document(self): setup_routing(self.app, routs=["contract_document_create"]) file_ = StringIO() file_.name = 'test_document.txt' file_.write("test upload contract document text data") file_.seek(0) doc = self.client.upload_document(file_, self.contract) self.assertEqual(doc.data.title, file_.name) self.assertEqual(doc.data.id, TEST_CONTRACT_KEYS.new_document_id) file_.close()
class ContractingUserTestCase(unittest.TestCase): """""" def setUp(self): #self._testMethodName self.app = Bottle() setup_routing(self.app) self.server = WSGIServer(('localhost', 20602), self.app, log=None) self.server.start() self.client = ContractingClient(API_KEY, host_url=HOST_URL, api_version=API_VERSION) with open(ROOT + 'contract_' + TEST_CONTRACT_KEYS.contract_id + '.json') as contract: self.contract = munchify(load(contract)) self.contract.update({'access':{"token": API_KEY}}) def tearDown(self): self.server.stop() ########################################################################### # CREATE ITEM TEST ########################################################################### def test_create_contract(self): setup_routing(self.app, routs=["contract_create"]) contract = munchify({'data': 'contract'}) self.client.create_contract(contract) ########################################################################### # DOCUMENTS FILE TEST ########################################################################### def test_upload_contract_document(self): setup_routing(self.app, routs=["contract_document_create"]) file_ = StringIO() file_.name = 'test_document.txt' file_.write("test upload contract document text data") file_.seek(0) doc = self.client.upload_document(file_, self.contract) self.assertEqual(doc.data.title, file_.name) self.assertEqual(doc.data.id, TEST_CONTRACT_KEYS.new_document_id) file_.close()
def setUp(self): # self._testMethodName self.app = Bottle() setup_routing(self.app) self.server = WSGIServer(("localhost", 20602), self.app, log=None) self.server.start() self.client = ContractingClient(API_KEY, host_url=HOST_URL, api_version=API_VERSION) with open(ROOT + "contract_" + TEST_CONTRACT_KEYS.contract_id + ".json") as contract: self.contract = munchify(load(contract)) self.contract.update({"access": {"token": API_KEY}})
def __init__(self, config): super(ContractingDataBridge, self).__init__() self.config = config self.contracting_client = ContractingClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version') ) params = {'opt_fields': 'status,lots', 'mode': '_all_'} self.tenders_client_forward = TendersClient( '', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), params=params ) params_desc = deepcopy(params) params_desc['descending'] = 1 self.tenders_client_backward = TendersClient( '', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), params=params_desc ) self.client = TendersClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.tenders_queue = Queue() self.handicap_contracts_queue = Queue() self.contracts_put_queue = Queue() self.contracts_retry_put_queue = Queue()
class ContractingDataBridge(object): """ Contracting Data Bridge """ def __init__(self, config): super(ContractingDataBridge, self).__init__() self.config = config self.tenders_sync_client = TendersClientSync('', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.client = TendersClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.contracting_client = ContractingClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version') ) self.initial_sync_point = {} self.tenders_queue = Queue(maxsize=500) self.handicap_contracts_queue = Queue(maxsize=500) self.contracts_put_queue = Queue(maxsize=500) self.contracts_retry_put_queue = Queue(maxsize=500) def config_get(self, name): return self.config.get('main').get(name) @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) def get_tender_credentials(self, tender_id): self.client.headers.update({'X-Client-Request-ID': generate_req_id()}) logger.info("Getting credentials for tender {}".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GET_CREDENTIALS}, {"TENDER_ID": tender_id})) data = self.client.extract_credentials(tender_id) logger.info("Got tender {} credentials".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GOT_CREDENTIALS}, {"TENDER_ID": tender_id})) return data @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) def initialize_sync(self, params=None, direction=None): # TODO use gevent.Event to wake up forward sync instead of checking # initial sync point if direction == "backward": assert params['descending'] response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) # set values in reverse order due to 'descending' option self.initial_sync_point = {'forward_offset': response.prev_page.offset, 'backward_offset': response.next_page.offset} logger.info("Initial sync point {}".format(self.initial_sync_point)) return response elif not self.initial_sync_point: raise ValueError else: assert 'descending' not in params params['offset'] = self.initial_sync_point['forward_offset'] logger.info("Starting forward sync from offset {}".format(params['offset'])) return self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) def get_tenders(self, params={}, direction=""): response = self.initialize_sync(params=params, direction=direction) while not (params.get('descending') and not len(response.data) and params.get('offset') == response.next_page.offset): tenders_list = response.data params['offset'] = response.next_page.offset delay = 101 if tenders_list: delay = 15 logger.info("Client {} params: {}".format(direction, params)) for tender in tenders_list: if tender['status'] in ("active.qualification", "active.awarded", "complete"): if hasattr(tender, "lots"): if any([1 for lot in tender['lots'] if lot['status'] == "complete"]): logger.info('{} sync: Found multilot tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_MULTILOT_COMPLETE}, {"TENDER_ID": tender['id']})) yield tender elif tender['status'] == "complete": logger.info('{} sync: Found tender in complete status {}'.format(direction.capitalize(), tender['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_NOLOT_COMPLETE}, {"TENDER_ID": tender['id']})) yield tender else: logger.debug('{} sync: Skipping tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']), extra=journal_context(params={"TENDER_ID": tender['id']})) logger.info('Sleep {} sync...'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_SLEEP})) gevent.sleep(delay) logger.info('Restore {} sync'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_RESUME})) logger.debug('{} {}'.format(direction, params)) response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) def get_tender_contracts(self): while True: try: tender_to_sync = self.tenders_queue.get() tender = self.tenders_sync_client.get_tender(tender_to_sync['id'], extra_headers={'X-Client-Request-ID': generate_req_id()})['data'] db.put(tender_to_sync['id'], {'dateModified': tender_to_sync['dateModified']}) except Exception, e: logger.exception(e) logger.info('Put tender {} back to tenders queue'.format(tender_to_sync['id']), extra=journal_context(params={"TENDER_ID": tender_to_sync['id']})) self.tenders_queue.put(tender_to_sync) else: if 'contracts' not in tender: logger.warn('!!!No contracts found in tender {}'.format(tender['id']), extra=journal_context(params={"TENDER_ID": tender['id']})) continue for contract in tender['contracts']: if contract["status"] == "active": try: if not db.has(contract['id']): self.contracting_client.get_contract(contract['id']) else: logger.info('Contract {} exists in local db'.format(contract['id']), extra=journal_context(params={"CONTRACT_ID": contract['id']})) continue except ResourceNotFound: logger.info('Sync contract {} of tender {}'.format(contract['id'], tender['id']), extra=journal_context( {"MESSAGE_ID": DATABRIDGE_CONTRACT_TO_SYNC}, {"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']})) except Exception, e: logger.exception(e) logger.info('Put tender {} back to tenders queue'.format(tender_to_sync['id']), extra=journal_context(params={"TENDER_ID": tender_to_sync['id']})) self.tenders_queue.put(tender_to_sync) break else: db.put(contract['id'], True) logger.info('Contract exists {}'.format(contract['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_CONTRACT_EXISTS}, {"CONTRACT_ID": contract['id']})) continue contract['tender_id'] = tender['id'] contract['procuringEntity'] = tender['procuringEntity'] if not contract.get('items'): logger.info('Copying contract {} items'.format(contract['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_COPY_CONTRACT_ITEMS}, {"CONTRACT_ID": contract['id']})) if tender.get('lots'): related_awards = [aw for aw in tender['awards'] if aw['id'] == contract['awardID']] if related_awards: award = related_awards[0] if award.get("items"): logger.debug('Copying items from related award {}'.format(award['id'])) contract['items'] = award['items'] else: logger.debug('Copying items matching related lot {}'.format(award['relatedLot'])) contract['items'] = [item for item in tender['items'] if item['relatedLot'] == award['lotID']] else: logger.warn('Not found related award for contact {} of tender {}'.format(contract['id'], tender['id']), extra=journal_context(params={"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']})) else: logger.debug('Copying all tender {} items into contract {}'.format(tender['id'], contract['id']), extra=journal_context(params={"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']})) contract['items'] = tender['items'] if not contract.get('items'): logger.warn('Contact {} of tender {} does not contain items info'.format(contract['id'], tender['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_MISSING_CONTRACT_ITEMS}, {"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']})) self.handicap_contracts_queue.put(contract)
class ContractingDataBridge(object): """ Contracting Data Bridge """ def __init__(self, config): super(ContractingDataBridge, self).__init__() self.config = config self.contracting_client = ContractingClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version') ) params = {'opt_fields': 'status,lots', 'mode': '_all_'} self.tenders_client_forward = TendersClient( '', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), params=params ) params_desc = deepcopy(params) params_desc['descending'] = 1 self.tenders_client_backward = TendersClient( '', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), params=params_desc ) self.client = TendersClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.tenders_queue = Queue() self.handicap_contracts_queue = Queue() self.contracts_put_queue = Queue() self.contracts_retry_put_queue = Queue() def config_get(self, name): return self.config.get('main').get(name) @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) def get_tender_credentials(self, tender_id): return self.client.extract_credentials(tender_id) def get_tenders(self, client): while True: request_id = generate_req_id() client.headers.update({'X-Client-Request-ID': request_id}) tenders_list = list(client.get_tenders()) delay = 101 if tenders_list: delay = 15 logger.info("Client params: {}".format(client.params)) for tender in tenders_list: if tender['status'] in ("active.qualification", "active.awarded", "complete"): if hasattr(tender, "lots"): if any([1 for lot in tender['lots'] if lot['status'] == "complete"]): logger.info('found multilot tender {} in status {}'.format(tender['id'], tender['status'])) yield tender elif tender['status'] == "complete": logger.info('Found tender in complete status {}'.format(tender['id'])) yield tender else: logger.debug('Skipping tender {} in status {}'.format(tender['id'], tender['status'])) logger.info('Sleep...') time.sleep(delay) def get_tender_contracts(self): while True: request_id = generate_req_id() self.tenders_client_backward.headers.update({'X-Client-Request-ID': request_id}) tender = self.tenders_client_backward.get_tender(self.tenders_queue.get()['id'])['data'] if 'contracts' not in tender: logger.warn('!!!No contracts found in tender {}'.format(tender['id'])) continue for contract in tender['contracts']: if contract["status"] == "active": try: self.contracting_client.get_contract(contract['id']) except ResourceNotFound: logger.info('Sync contract {} of tender {}'.format(contract['id'], tender['id'])) else: logger.info('Contract exists {}'.format(contract['id'])) continue contract['tender_id'] = tender['id'] contract['procuringEntity'] = tender['procuringEntity'] self.handicap_contracts_queue.put(contract) def prepare_contract_data(self): while True: contract = self.handicap_contracts_queue.get() try: logger.info("Getting extra info for tender {}".format(contract['tender_id'])) tender_data = self.get_tender_credentials(contract['tender_id']) except Exception, e: logger.info("Can't get tender credentials {}".format(contract['tender_id'])) self.handicap_contracts_queue.put(contract) else: logger.debug("Got extra info for tender {}".format( contract['tender_id'])) data = tender_data.data if data.get('mode'): contract['mode'] = data['mode'] contract['owner'] = data['owner'] contract['tender_token'] = data['tender_token'] del contract['status'] # create contract in 'draft' status self.contracts_put_queue.put(contract) gevent.sleep(0)
def __init__(self, config): super(ContractingDataBridge, self).__init__() self.config = config self.cache_db = Db(self.config.get('main')) self._backend = "redis" self._host = self.config.get('cache_host') self._port = self.config.get('cache_port') or 6379 self._db_name = self.config.get('cache_db_name') or 0 logger.info( "Caching backend: '{}', db name: '{}', host: '{}', port: '{}'". format(self.cache_db._backend, self.cache_db._db_name, self.cache_db._host, self.cache_db._port), extra=journal_context({"MESSAGE_ID": DATABRIDGE_INFO}, {})) self.on_error_delay = self.config_get('on_error_sleep_delay') or 5 self.jobs_watcher_delay = self.config_get('jobs_watcher_delay') or 15 queue_size = self.config_get('buffers_size') or 500 self.full_stack_sync_delay = self.config_get( 'full_stack_sync_delay') or 15 self.empty_stack_sync_delay = self.config_get( 'empty_stack_sync_delay') or 101 api_server = self.config_get('tenders_api_server') api_version = self.config_get('tenders_api_version') ro_api_server = self.config_get( 'public_tenders_api_server') or api_server contracting_api_server = self.config_get('contracting_api_server') contracting_api_version = self.config_get('contracting_api_version') self.tenders_sync_client = TendersClientSync( '', host_url=ro_api_server, api_version=api_version, ) self.client = TendersClient( self.config_get('api_token'), host_url=api_server, api_version=api_version, ) self.contracting_client = ContractingClient( self.config_get('api_token'), host_url=contracting_api_server, api_version=contracting_api_version) self.contracting_client_ro = self.contracting_client if self.config_get('public_tenders_api_server'): if api_server == contracting_api_server and api_version == contracting_api_version: self.contracting_client_ro = ContractingClient( self.config_get('api_token'), host_url=ro_api_server, api_version=api_version) self.initial_sync_point = {} self.initialization_event = gevent.event.Event() self.tenders_queue = Queue(maxsize=queue_size) self.handicap_contracts_queue = Queue(maxsize=queue_size) self.contracts_put_queue = Queue(maxsize=queue_size) self.contracts_retry_put_queue = Queue(maxsize=queue_size) self.basket = {}