class ContractingDataBridge(object): """ Contracting Data Bridge """ def __init__(self, config): super(ContractingDataBridge, self).__init__() self.config = config self.tenders_sync_client = TendersClientSync('', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.client = TendersClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.contracting_client = ContractingClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version') ) self.initial_sync_point = {} self.tenders_queue = Queue(maxsize=500) self.handicap_contracts_queue = Queue(maxsize=500) self.contracts_put_queue = Queue(maxsize=500) self.contracts_retry_put_queue = Queue(maxsize=500) def config_get(self, name): return self.config.get('main').get(name) @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) def get_tender_credentials(self, tender_id): self.client.headers.update({'X-Client-Request-ID': generate_req_id()}) logger.info("Getting credentials for tender {}".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GET_CREDENTIALS}, {"TENDER_ID": tender_id})) data = self.client.extract_credentials(tender_id) logger.info("Got tender {} credentials".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GOT_CREDENTIALS}, {"TENDER_ID": tender_id})) return data @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) def initialize_sync(self, params=None, direction=None): # TODO use gevent.Event to wake up forward sync instead of checking # initial sync point if direction == "backward": assert params['descending'] response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) # set values in reverse order due to 'descending' option self.initial_sync_point = {'forward_offset': response.prev_page.offset, 'backward_offset': response.next_page.offset} logger.info("Initial sync point {}".format(self.initial_sync_point)) return response elif not self.initial_sync_point: raise ValueError else: assert 'descending' not in params params['offset'] = self.initial_sync_point['forward_offset'] logger.info("Starting forward sync from offset {}".format(params['offset'])) return self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) def get_tenders(self, params={}, direction=""): response = self.initialize_sync(params=params, direction=direction) while not (params.get('descending') and not len(response.data) and params.get('offset') == response.next_page.offset): tenders_list = response.data params['offset'] = response.next_page.offset delay = 101 if tenders_list: delay = 15 logger.info("Client {} params: {}".format(direction, params)) for tender in tenders_list: if tender['status'] in ("active.qualification", "active.awarded", "complete"): if hasattr(tender, "lots"): if any([1 for lot in tender['lots'] if lot['status'] == "complete"]): logger.info('{} sync: Found multilot tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_MULTILOT_COMPLETE}, {"TENDER_ID": tender['id']})) yield tender elif tender['status'] == "complete": logger.info('{} sync: Found tender in complete status {}'.format(direction.capitalize(), tender['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_NOLOT_COMPLETE}, {"TENDER_ID": tender['id']})) yield tender else: logger.debug('{} sync: Skipping tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']), extra=journal_context(params={"TENDER_ID": tender['id']})) logger.info('Sleep {} sync...'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_SLEEP})) gevent.sleep(delay) logger.info('Restore {} sync'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_RESUME})) logger.debug('{} {}'.format(direction, params)) response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) def get_tender_contracts(self): while True: try: tender_to_sync = self.tenders_queue.get() tender = self.tenders_sync_client.get_tender(tender_to_sync['id'], extra_headers={'X-Client-Request-ID': generate_req_id()})['data'] db.put(tender_to_sync['id'], {'dateModified': tender_to_sync['dateModified']}) except Exception, e: logger.exception(e) logger.info('Put tender {} back to tenders queue'.format(tender_to_sync['id']), extra=journal_context(params={"TENDER_ID": tender_to_sync['id']})) self.tenders_queue.put(tender_to_sync) else: if 'contracts' not in tender: logger.warn('!!!No contracts found in tender {}'.format(tender['id']), extra=journal_context(params={"TENDER_ID": tender['id']})) continue for contract in tender['contracts']: if contract["status"] == "active": try: if not db.has(contract['id']): self.contracting_client.get_contract(contract['id']) else: logger.info('Contract {} exists in local db'.format(contract['id']), extra=journal_context(params={"CONTRACT_ID": contract['id']})) continue except ResourceNotFound: logger.info('Sync contract {} of tender {}'.format(contract['id'], tender['id']), extra=journal_context( {"MESSAGE_ID": DATABRIDGE_CONTRACT_TO_SYNC}, {"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']})) except Exception, e: logger.exception(e) logger.info('Put tender {} back to tenders queue'.format(tender_to_sync['id']), extra=journal_context(params={"TENDER_ID": tender_to_sync['id']})) self.tenders_queue.put(tender_to_sync) break else: db.put(contract['id'], True) logger.info('Contract exists {}'.format(contract['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_CONTRACT_EXISTS}, {"CONTRACT_ID": contract['id']})) continue contract['tender_id'] = tender['id'] contract['procuringEntity'] = tender['procuringEntity'] if not contract.get('items'): logger.info('Copying contract {} items'.format(contract['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_COPY_CONTRACT_ITEMS}, {"CONTRACT_ID": contract['id']})) if tender.get('lots'): related_awards = [aw for aw in tender['awards'] if aw['id'] == contract['awardID']] if related_awards: award = related_awards[0] if award.get("items"): logger.debug('Copying items from related award {}'.format(award['id'])) contract['items'] = award['items'] else: logger.debug('Copying items matching related lot {}'.format(award['relatedLot'])) contract['items'] = [item for item in tender['items'] if item['relatedLot'] == award['lotID']] else: logger.warn('Not found related award for contact {} of tender {}'.format(contract['id'], tender['id']), extra=journal_context(params={"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']})) else: logger.debug('Copying all tender {} items into contract {}'.format(tender['id'], contract['id']), extra=journal_context(params={"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']})) contract['items'] = tender['items'] if not contract.get('items'): logger.warn('Contact {} of tender {} does not contain items info'.format(contract['id'], tender['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_MISSING_CONTRACT_ITEMS}, {"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']})) self.handicap_contracts_queue.put(contract)
class ContractingDataBridge(object): """ Contracting Data Bridge """ def __init__(self, config): super(ContractingDataBridge, self).__init__() self.config = config self.contracting_client = ContractingClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version') ) params = {'opt_fields': 'status,lots', 'mode': '_all_'} self.tenders_client_forward = TendersClient( '', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), params=params ) params_desc = deepcopy(params) params_desc['descending'] = 1 self.tenders_client_backward = TendersClient( '', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), params=params_desc ) self.client = TendersClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.tenders_queue = Queue() self.handicap_contracts_queue = Queue() self.contracts_put_queue = Queue() self.contracts_retry_put_queue = Queue() def config_get(self, name): return self.config.get('main').get(name) @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) def get_tender_credentials(self, tender_id): return self.client.extract_credentials(tender_id) def get_tenders(self, client): while True: request_id = generate_req_id() client.headers.update({'X-Client-Request-ID': request_id}) tenders_list = list(client.get_tenders()) delay = 101 if tenders_list: delay = 15 logger.info("Client params: {}".format(client.params)) for tender in tenders_list: if tender['status'] in ("active.qualification", "active.awarded", "complete"): if hasattr(tender, "lots"): if any([1 for lot in tender['lots'] if lot['status'] == "complete"]): logger.info('found multilot tender {} in status {}'.format(tender['id'], tender['status'])) yield tender elif tender['status'] == "complete": logger.info('Found tender in complete status {}'.format(tender['id'])) yield tender else: logger.debug('Skipping tender {} in status {}'.format(tender['id'], tender['status'])) logger.info('Sleep...') time.sleep(delay) def get_tender_contracts(self): while True: request_id = generate_req_id() self.tenders_client_backward.headers.update({'X-Client-Request-ID': request_id}) tender = self.tenders_client_backward.get_tender(self.tenders_queue.get()['id'])['data'] if 'contracts' not in tender: logger.warn('!!!No contracts found in tender {}'.format(tender['id'])) continue for contract in tender['contracts']: if contract["status"] == "active": try: self.contracting_client.get_contract(contract['id']) except ResourceNotFound: logger.info('Sync contract {} of tender {}'.format(contract['id'], tender['id'])) else: logger.info('Contract exists {}'.format(contract['id'])) continue contract['tender_id'] = tender['id'] contract['procuringEntity'] = tender['procuringEntity'] self.handicap_contracts_queue.put(contract) def prepare_contract_data(self): while True: contract = self.handicap_contracts_queue.get() try: logger.info("Getting extra info for tender {}".format(contract['tender_id'])) tender_data = self.get_tender_credentials(contract['tender_id']) except Exception, e: logger.info("Can't get tender credentials {}".format(contract['tender_id'])) self.handicap_contracts_queue.put(contract) else: logger.debug("Got extra info for tender {}".format( contract['tender_id'])) data = tender_data.data if data.get('mode'): contract['mode'] = data['mode'] contract['owner'] = data['owner'] contract['tender_token'] = data['tender_token'] del contract['status'] # create contract in 'draft' status self.contracts_put_queue.put(contract) gevent.sleep(0)