def __init__(self, key, config): params = dict() if config['mode'] in ('test', '_all_'): params['mode'] = config['mode'] TendersClient.__init__(self, key, config['host_url'], config['api_version'], params) if config.get('resource'): self.prefix_path = '/api/{}/{}'.format(config['api_version'], config['resource']) if config['timeout']: socket.setdefaulttimeout(float(config['timeout']))
def __init__(self, key, config): params = {'limit': 1000} if config['mode'] in ('test', '_all_'): params['mode'] = config['mode'] if config['timeout']: socket.setdefaulttimeout(float(config['timeout'])) TendersClient.__init__(self, key, config['host_url'], config['api_version'], params) if config.get('resource'): self.prefix_path = '/api/{}/{}'.format(config['api_version'], config['resource']) self.allow_preload = config.get('preload', None) self.api_version = config['api_version']
def __init__(self, key, config): params = dict() if config['mode'] in ('test', '_all_'): params['mode'] = config['mode'] TendersClient.__init__(self, key, config['host_url'], config['api_version'], params) if config.get('resource'): self.prefix_path = '/api/{}/{}'.format(config['api_version'], config['resource']) if config['timeout']: socket.setdefaulttimeout(float(config['timeout']))
def clients_initialize(self): self.client = TendersClient( self.config_get('api_token'), host_url=self.api_server, api_version=self.api_version, ) self.contracting_client_init() self.tenders_sync_client = TendersClientSync('', host_url=self.ro_api_server, api_version=self.api_version, )
def __init__(self, config): super(EdgeDataBridge, self).__init__() self.config = config self.api_host = self.config_get('tenders_api_server') self.api_version = self.config_get('tenders_api_version') self.client = TendersClient(host_url=self.api_host, api_version=self.api_version, key='') self.couch_url = urljoin(self.config_get('couch_url'), self.config_get('public_db')) self.db = Database(self.couch_url, session=Session(retry_delays=range(10)))
def __init__(self, config): super(ContractingDataBridge, self).__init__() self.config = config self.tenders_sync_client = TendersClientSync('', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.client = TendersClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.contracting_client = ContractingClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version') ) self.initial_sync_point = {} self.tenders_queue = Queue(maxsize=500) self.handicap_contracts_queue = Queue(maxsize=500) self.contracts_put_queue = Queue(maxsize=500) self.contracts_retry_put_queue = Queue(maxsize=500)
class EdgeDataBridge(object): """Edge Bridge""" def __init__(self, config): super(EdgeDataBridge, self).__init__() self.config = config self.api_host = self.config_get('tenders_api_server') self.api_version = self.config_get('tenders_api_version') self.retrievers_params = self.config_get('retrievers_params') self.client = TendersClient(host_url=self.api_host, api_version=self.api_version, key='' ) self.couch_url = urljoin( self.config_get('couch_url'), self.config_get('public_db') ) self.db = Database(self.couch_url, session=Session(retry_delays=range(10))) def config_get(self, name): return self.config.get('main').get(name) def get_teders_list(self): for item in get_tenders(host=self.api_host, version=self.api_version, key='', extra_params={'mode': '_all_'}, retrievers_params=self.retrievers_params): yield (item["id"], item["dateModified"]) def save_tender_in_db(self, tender_id, date_modified): tender_doc = self.db.get(tender_id) if tender_doc: if tender_doc['dateModified'] == date_modified: return tender = self.client.get_tender(tender_id).get('data') if tender: tender['_id'] = tender_id tender['doc_type'] = 'Tender' if tender_doc: tender['_rev'] = tender_doc['_rev'] logger.info('Update tender {} '.format(tender_id)) else: logger.info('Save tender {} '.format(tender_id)) try: self.db.save(tender) except Exception as e: logger.info('Saving tender {} fail with error {}'.format(tender_id, e.message), extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'}) else: logger.info('Tender {} not found'.format(tender_id)) def run(self): logger.info('Start Edge Bridge', extra={'MESSAGE_ID': 'edge_bridge_start_bridge'}) logger.info('Start data sync...', extra={'MESSAGE_ID': 'edge_bridge__data_sync'}) for tender_id, date_modified in self.get_teders_list(): self.save_tender_in_db(tender_id, date_modified)
class EdgeDataBridge(object): """Edge Bridge""" def __init__(self, config): super(EdgeDataBridge, self).__init__() self.config = config self.api_host = self.config_get('tenders_api_server') self.api_version = self.config_get('tenders_api_version') self.client = TendersClient(host_url=self.api_host, api_version=self.api_version, key='') self.couch_url = urljoin(self.config_get('couch_url'), self.config_get('public_db')) self.db = Database(self.couch_url, session=Session(retry_delays=range(10))) def config_get(self, name): return self.config.get('main').get(name) def get_teders_list(self): for item in get_tenders(host=self.api_host, version=self.api_version, key='', extra_params={'mode': '_all_'}): yield (item["id"], item["dateModified"]) def save_tender_in_db(self, tender_id, date_modified): tender_doc = self.db.get(tender_id) if tender_doc: if tender_doc['dateModified'] == date_modified: return tender = self.client.get_tender(tender_id).get('data') if tender: tender['_id'] = tender_id tender['doc_type'] = 'Tender' if tender_doc: tender['_rev'] = tender_doc['_rev'] logger.info('Update tender {} '.format(tender_id)) else: logger.info('Save tender {} '.format(tender_id)) try: self.db.save(tender) except Exception as e: logger.info( 'Saving tender {} fail with error {}'.format( tender_id, e.message), extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'}) else: logger.info('Tender {} not found'.format(tender_id)) def run(self): logger.info('Start Edge Bridge', extra={'MESSAGE_ID': 'edge_bridge_start_bridge'}) logger.info('Start data sync...', extra={'MESSAGE_ID': 'edge_bridge__data_sync'}) for tender_id, date_modified in self.get_teders_list(): self.save_tender_in_db(tender_id, date_modified)
def __init__(self, config): super(Oracle, self).__init__() self.config = config self.api_host = self.config_get('tenders_api_server') self.api_version = self.config_get('tenders_api_version') self.retrievers_params = self.config_get('retrievers_params') self.db = DB try: self.client = TendersClient(host_url=self.api_host, api_version=self.api_version, key='') except MissingSchema: raise ConfigError( 'In config dictionary empty or missing \'tenders_api_server\'') except ConnectionError as e: raise e
def get_tender(self, tender_id): for i in range(5): try: resp = TendersClient.get_tender(self, tender_id) except ResourceError as e: logger.error("get_tender on %s error %s", tender_id, e) time.sleep(10) else: return resp raise ResourceError("Maximum retry reached")
def get_tender(self, tender_id): for i in range(5): try: resp = TendersClient.get_tender(self, tender_id) except ResourceError as e: logger.error("get_tender on %s error %s", tender_id, e) time.sleep(10) else: return resp raise ResourceError("Maximum retry reached")
def __init__(self, config): super(EdgeDataBridge, self).__init__() self.config = config self.api_host = self.config_get('tenders_api_server') self.api_version = self.config_get('tenders_api_version') self.retrievers_params = self.config_get('retrievers_params') try: self.client = TendersClient(host_url=self.api_host, api_version=self.api_version, key='') except MissingSchema: raise DataBridgeConfigError( 'In config dictionary empty or missing \'tenders_api_server\'') except ConnectionError as e: raise e self.couch_url = urljoin(self.config_get('couch_url'), self.config_get('public_db')) self.db = Database(self.couch_url, session=Session(retry_delays=range(10))) try: self.db.info() except ResourceNotFound: error_message = "Database with name '" + self.config_get( 'public_db') + "' doesn\'t exist" raise DataBridgeConfigError(error_message) except error as e: if e.errno == errno.ECONNREFUSED: raise DataBridgeConfigError( "Connection refused: 'couch_url' is invalid in config dictionary" ) except AttributeError as e: raise DataBridgeConfigError( '\'couch_url\' is missed or empty in config dictionary.') except KeyError as e: if e.message == 'db_name': raise DataBridgeConfigError( '\'public_db\' name is missed or empty in config dictionary' )
def get_tender(self, tender_id): for i in range(5): try: if not self.headers.get('Cookie', None): self.request_cookie() return TendersClient.get_tender(self, tender_id) except (socket.error, ResourceError) as e: logger.error("get_tender %s reason %s", tender_id, str(e)) if i > 1: self.headers.pop('Cookie', None) #self.params.pop('offset', None) time.sleep(10 * i + 10) raise ResourceError("Maximum retry reached")
def __init__(self, config): super(ContractingDataBridge, self).__init__() self.config = config self.contracting_client = ContractingClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version') ) params = {'opt_fields': 'status,lots', 'mode': '_all_'} self.tenders_client_forward = TendersClient( '', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), params=params ) params_desc = deepcopy(params) params_desc['descending'] = 1 self.tenders_client_backward = TendersClient( '', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), params=params_desc ) self.client = TendersClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.tenders_queue = Queue() self.handicap_contracts_queue = Queue() self.contracts_put_queue = Queue() self.contracts_retry_put_queue = Queue()
def __init__(self, config): super(EdgeDataBridge, self).__init__() self.config = config self.api_host = self.config_get('tenders_api_server') self.api_version = self.config_get('tenders_api_version') self.retrievers_params = self.config_get('retrievers_params') self.client = TendersClient(host_url=self.api_host, api_version=self.api_version, key='' ) self.couch_url = urljoin( self.config_get('couch_url'), self.config_get('public_db') ) self.db = Database(self.couch_url, session=Session(retry_delays=range(10)))
def validate_credentials_generate(request): try: token = get_access_token(request) except ValueError: raise_operation_error(request, 'No access token was provided.') try: response = TendersClient( request.registry.api_token, host_url=request.registry.api_server, api_version=request.registry.api_version, ).extract_credentials(request.validated['monitoring'].tender_id) except ResourceNotFound: raise_operation_error( request, 'Tender {} not found'.format( request.validated['monitoring'].tender_id)) else: if sha512(token).hexdigest() != response['data']['tender_token']: raise forbidden(request)
class ContractingDataBridge(object): """ Contracting Data Bridge """ def __init__(self, config): super(ContractingDataBridge, self).__init__() self.config = config self.cache_db = Db(self.config.get('main')) self._backend = "redis" self._host = self.config.get('cache_host') self._port = self.config.get('cache_port') or 6379 self._db_name = self.config.get('cache_db_name') or 0 logger.info("Caching backend: '{}', db name: '{}', host: '{}', port: '{}'".format(self.cache_db._backend, self.cache_db._db_name, self.cache_db._host, self.cache_db._port), extra=journal_context({"MESSAGE_ID": DATABRIDGE_INFO}, {})) self.on_error_delay = self.config_get('on_error_sleep_delay') or 5 self.jobs_watcher_delay = self.config_get('jobs_watcher_delay') or 15 queue_size = self.config_get('buffers_size') or 500 self.full_stack_sync_delay = self.config_get('full_stack_sync_delay') or 15 self.empty_stack_sync_delay = self.config_get('empty_stack_sync_delay') or 101 self.api_server = self.config_get('tenders_api_server') self.api_version = self.config_get('tenders_api_version') self.ro_api_server = self.config_get('public_tenders_api_server') or self.api_server self.contracting_api_server = self.config_get('contracting_api_server') self.contracting_api_version = self.config_get('contracting_api_version') self.clients_initialize() self.initial_sync_point = {} self.initialization_event = gevent.event.Event() self.tenders_queue = Queue(maxsize=queue_size) self.handicap_contracts_queue = Queue(maxsize=queue_size) self.contracts_put_queue = Queue(maxsize=queue_size) self.contracts_retry_put_queue = Queue(maxsize=queue_size) self.basket = {} def contracting_client_init(self): logger.info('Initialization contracting clients.', extra=journal_context({"MESSAGE_ID": DATABRIDGE_INFO}, {})) self.contracting_client = ContractingClient( self.config_get('api_token'), host_url=self.contracting_api_server, api_version=self.contracting_api_version ) self.contracting_client_ro = self.contracting_client if self.config_get('public_tenders_api_server'): if self.api_server == self.contracting_api_server and self.api_version == self.contracting_api_version: self.contracting_client_ro = ContractingClient( '', host_url=self.ro_api_server, api_version=self.api_version ) def clients_initialize(self): self.client = TendersClient( self.config_get('api_token'), host_url=self.api_server, api_version=self.api_version, ) self.contracting_client_init() self.tenders_sync_client = TendersClientSync('', host_url=self.ro_api_server, api_version=self.api_version, ) def config_get(self, name): return self.config.get('main').get(name) @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) def get_tender_credentials(self, tender_id): self.client.headers.update({'X-Client-Request-ID': generate_req_id()}) logger.info("Getting credentials for tender {}".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GET_CREDENTIALS}, {"TENDER_ID": tender_id})) data = self.client.extract_credentials(tender_id) logger.info("Got tender {} credentials".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GOT_CREDENTIALS}, {"TENDER_ID": tender_id})) return data def initialize_sync(self, params=None, direction=None): self.initialization_event.clear() if direction == "backward": assert params['descending'] response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) # set values in reverse order due to 'descending' option self.initial_sync_point = {'forward_offset': response.prev_page.offset, 'backward_offset': response.next_page.offset} self.initialization_event.set() # wake up forward worker logger.info("Initial sync point {}".format(self.initial_sync_point)) return response else: assert 'descending' not in params gevent.wait([self.initialization_event]) params['offset'] = self.initial_sync_point['forward_offset'] logger.info("Starting forward sync from offset {}".format(params['offset'])) return self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) def get_tenders(self, params={}, direction=""): response = self.initialize_sync(params=params, direction=direction) while not (params.get('descending') and not len(response.data) and params.get('offset') == response.next_page.offset): tenders_list = response.data params['offset'] = response.next_page.offset delay = self.empty_stack_sync_delay if tenders_list: delay = self.full_stack_sync_delay logger.info("Client {} params: {}".format(direction, params)) for tender in tenders_list: if tender.get('procurementMethodType') in ['competitiveDialogueUA', 'competitiveDialogueEU']: logger.info('Skipping {} tender {}'.format(tender['procurementMethodType'], tender['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_INFO}, params={"TENDER_ID": tender['id']})) continue if tender['status'] in ("active.qualification", "active", "active.awarded", "complete"): if hasattr(tender, "lots"): if any([1 for lot in tender['lots'] if lot['status'] == "complete"]): logger.info('{} sync: Found multilot tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_MULTILOT_COMPLETE}, {"TENDER_ID": tender['id']})) yield tender elif tender['status'] == "complete": logger.info('{} sync: Found tender in complete status {}'.format(direction.capitalize(), tender['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_NOLOT_COMPLETE}, {"TENDER_ID": tender['id']})) yield tender else: logger.debug('{} sync: Skipping tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']), extra=journal_context(params={"TENDER_ID": tender['id']})) logger.info('Sleep {} sync...'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_SLEEP})) gevent.sleep(delay) logger.info('Restore {} sync'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_RESUME})) logger.debug('{} {}'.format(direction, params)) response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) def _put_tender_in_cache_by_contract(self, contract, tender_id): dateModified = self.basket.get(contract['id']) if dateModified: # TODO: save tender in cache only if all active contracts are # handled successfully self.cache_db.put(tender_id, dateModified) self.basket.pop(contract['id'], None) def _get_tender_contracts(self): try: tender_to_sync = self.tenders_queue.get() tender = self.tenders_sync_client.get_tender(tender_to_sync['id'], extra_headers={'X-Client-Request-ID': generate_req_id()})['data'] except Exception, e: logger.warn('Fail to get tender info {}'.format(tender_to_sync['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_EXCEPTION}, params={"TENDER_ID": tender_to_sync['id']})) logger.exception(e) logger.info('Put tender {} back to tenders queue'.format(tender_to_sync['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_EXCEPTION}, params={"TENDER_ID": tender_to_sync['id']})) self.tenders_queue.put(tender_to_sync) gevent.sleep(self.on_error_delay) else:
class EdgeDataBridge(object): """Edge Bridge""" def __init__(self, config): super(EdgeDataBridge, self).__init__() self.config = config self.api_host = self.config_get('tenders_api_server') self.api_version = self.config_get('tenders_api_version') self.retrievers_params = self.config_get('retrievers_params') try: self.client = TendersClient(host_url=self.api_host, api_version=self.api_version, key='') except MissingSchema: raise DataBridgeConfigError( 'In config dictionary empty or missing \'tenders_api_server\'') except ConnectionError as e: raise e self.couch_url = urljoin(self.config_get('couch_url'), self.config_get('public_db')) self.db = Database(self.couch_url, session=Session(retry_delays=range(10))) try: self.db.info() except ResourceNotFound: error_message = "Database with name '" + self.config_get( 'public_db') + "' doesn\'t exist" raise DataBridgeConfigError(error_message) except error as e: if e.errno == errno.ECONNREFUSED: raise DataBridgeConfigError( "Connection refused: 'couch_url' is invalid in config dictionary" ) except AttributeError as e: raise DataBridgeConfigError( '\'couch_url\' is missed or empty in config dictionary.') except KeyError as e: if e.message == 'db_name': raise DataBridgeConfigError( '\'public_db\' name is missed or empty in config dictionary' ) def config_get(self, name): try: return self.config.get('main').get(name) except AttributeError as e: raise DataBridgeConfigError( 'In config dictionary missed section \'main\'') def get_teders_list(self): for item in get_tenders(host=self.api_host, version=self.api_version, key='', extra_params={'mode': '_all_'}, retrievers_params=self.retrievers_params): yield (item["id"], item["dateModified"]) def save_tender_in_db(self, tender_id, date_modified): tender_doc = self.db.get(tender_id) if tender_doc: if tender_doc['dateModified'] == date_modified: return tender = self.client.get_tender(tender_id).get('data') if tender: tender['_id'] = tender_id tender['doc_type'] = 'Tender' if tender_doc: tender['_rev'] = tender_doc['_rev'] logger.info('Update tender {} '.format(tender_id)) else: logger.info('Save tender {} '.format(tender_id)) try: self.db.save(tender) except Exception as e: logger.info( 'Saving tender {} fail with error {}'.format( tender_id, e.message), extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'}) else: logger.info('Tender {} not found'.format(tender_id)) def run(self): logger.info('Start Edge Bridge', extra={'MESSAGE_ID': 'edge_bridge_start_bridge'}) logger.info('Start data sync...', extra={'MESSAGE_ID': 'edge_bridge__data_sync'}) for tender_id, date_modified in self.get_teders_list(): self.save_tender_in_db(tender_id, date_modified)
class Oracle(object): def __init__(self, config): super(Oracle, self).__init__() self.config = config self.api_host = self.config_get('tenders_api_server') self.api_version = self.config_get('tenders_api_version') self.retrievers_params = self.config_get('retrievers_params') self.db = DB try: self.client = TendersClient(host_url=self.api_host, api_version=self.api_version, key='') except MissingSchema: raise ConfigError( 'In config dictionary empty or missing \'tenders_api_server\'') except ConnectionError as e: raise e def config_get(self, name): try: return self.config.get('main').get(name) except AttributeError as e: raise ConfigError('In config dictionary missed section \'main\'') def get_teders_list(self): for item in get_tenders(host=self.api_host, version=self.api_version, key='', extra_params={ 'mode': '_all_', 'opt_fields': 'status' }, retrievers_params=self.retrievers_params): yield (item["id"], item["status"]) def approve_bids(self, tender_id, date_modified): # TODO: At this point must be get_bids if self.db[tender['id']] == tender['status']: continue tender = self.client.get_tender(tender_id).get('data') if not tender: logger.info('Tender {} not found'.format(tender_id)) continue for bid in tender['bids']: doc_id = "{}_{}".format(tender['id'], bid['id']) if self.db[doc_id] == tender['status']: continue shard_key = "Oracle approve: " + bid['key'] self.client.patch_bid(tender['id'], bid['id'], {'data': { "shard": shard_key }}) self.db[doc_id] = tender['status'] self.db[tender['id']] = tender['status'] def run(self): logger.info('Start Oracle', extra={'MESSAGE_ID': 'start_oracle'}) logger.info('Start data sync...', extra={'MESSAGE_ID': 'oracle__data_sync'}) for tender_id, status in self.get_teders_list(): self.aporove_bids(tender_id, date_modified)
class ContractingDataBridge(object): """ Contracting Data Bridge """ def __init__(self, config): super(ContractingDataBridge, self).__init__() self.config = config self.contracting_client = ContractingClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version') ) params = {'opt_fields': 'status,lots', 'mode': '_all_'} self.tenders_client_forward = TendersClient( '', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), params=params ) params_desc = deepcopy(params) params_desc['descending'] = 1 self.tenders_client_backward = TendersClient( '', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), params=params_desc ) self.client = TendersClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.tenders_queue = Queue() self.handicap_contracts_queue = Queue() self.contracts_put_queue = Queue() self.contracts_retry_put_queue = Queue() def config_get(self, name): return self.config.get('main').get(name) @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) def get_tender_credentials(self, tender_id): return self.client.extract_credentials(tender_id) def get_tenders(self, client): while True: request_id = generate_req_id() client.headers.update({'X-Client-Request-ID': request_id}) tenders_list = list(client.get_tenders()) delay = 101 if tenders_list: delay = 15 logger.info("Client params: {}".format(client.params)) for tender in tenders_list: if tender['status'] in ("active.qualification", "active.awarded", "complete"): if hasattr(tender, "lots"): if any([1 for lot in tender['lots'] if lot['status'] == "complete"]): logger.info('found multilot tender {} in status {}'.format(tender['id'], tender['status'])) yield tender elif tender['status'] == "complete": logger.info('Found tender in complete status {}'.format(tender['id'])) yield tender else: logger.debug('Skipping tender {} in status {}'.format(tender['id'], tender['status'])) logger.info('Sleep...') time.sleep(delay) def get_tender_contracts(self): while True: request_id = generate_req_id() self.tenders_client_backward.headers.update({'X-Client-Request-ID': request_id}) tender = self.tenders_client_backward.get_tender(self.tenders_queue.get()['id'])['data'] if 'contracts' not in tender: logger.warn('!!!No contracts found in tender {}'.format(tender['id'])) continue for contract in tender['contracts']: if contract["status"] == "active": try: self.contracting_client.get_contract(contract['id']) except ResourceNotFound: logger.info('Sync contract {} of tender {}'.format(contract['id'], tender['id'])) else: logger.info('Contract exists {}'.format(contract['id'])) continue contract['tender_id'] = tender['id'] contract['procuringEntity'] = tender['procuringEntity'] self.handicap_contracts_queue.put(contract) def prepare_contract_data(self): while True: contract = self.handicap_contracts_queue.get() try: logger.info("Getting extra info for tender {}".format(contract['tender_id'])) tender_data = self.get_tender_credentials(contract['tender_id']) except Exception, e: logger.info("Can't get tender credentials {}".format(contract['tender_id'])) self.handicap_contracts_queue.put(contract) else: logger.debug("Got extra info for tender {}".format( contract['tender_id'])) data = tender_data.data if data.get('mode'): contract['mode'] = data['mode'] contract['owner'] = data['owner'] contract['tender_token'] = data['tender_token'] del contract['status'] # create contract in 'draft' status self.contracts_put_queue.put(contract) gevent.sleep(0)
class ContractingDataBridge(object): """ Contracting Data Bridge """ def __init__(self, config): super(ContractingDataBridge, self).__init__() self.config = config self.tenders_sync_client = TendersClientSync('', host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.client = TendersClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version'), ) self.contracting_client = ContractingClient( self.config_get('api_token'), host_url=self.config_get('tenders_api_server'), api_version=self.config_get('tenders_api_version') ) self.initial_sync_point = {} self.tenders_queue = Queue(maxsize=500) self.handicap_contracts_queue = Queue(maxsize=500) self.contracts_put_queue = Queue(maxsize=500) self.contracts_retry_put_queue = Queue(maxsize=500) def config_get(self, name): return self.config.get('main').get(name) @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) def get_tender_credentials(self, tender_id): self.client.headers.update({'X-Client-Request-ID': generate_req_id()}) logger.info("Getting credentials for tender {}".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GET_CREDENTIALS}, {"TENDER_ID": tender_id})) data = self.client.extract_credentials(tender_id) logger.info("Got tender {} credentials".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GOT_CREDENTIALS}, {"TENDER_ID": tender_id})) return data @retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000) def initialize_sync(self, params=None, direction=None): # TODO use gevent.Event to wake up forward sync instead of checking # initial sync point if direction == "backward": assert params['descending'] response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) # set values in reverse order due to 'descending' option self.initial_sync_point = {'forward_offset': response.prev_page.offset, 'backward_offset': response.next_page.offset} logger.info("Initial sync point {}".format(self.initial_sync_point)) return response elif not self.initial_sync_point: raise ValueError else: assert 'descending' not in params params['offset'] = self.initial_sync_point['forward_offset'] logger.info("Starting forward sync from offset {}".format(params['offset'])) return self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) def get_tenders(self, params={}, direction=""): response = self.initialize_sync(params=params, direction=direction) while not (params.get('descending') and not len(response.data) and params.get('offset') == response.next_page.offset): tenders_list = response.data params['offset'] = response.next_page.offset delay = 101 if tenders_list: delay = 15 logger.info("Client {} params: {}".format(direction, params)) for tender in tenders_list: if tender['status'] in ("active.qualification", "active.awarded", "complete"): if hasattr(tender, "lots"): if any([1 for lot in tender['lots'] if lot['status'] == "complete"]): logger.info('{} sync: Found multilot tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_MULTILOT_COMPLETE}, {"TENDER_ID": tender['id']})) yield tender elif tender['status'] == "complete": logger.info('{} sync: Found tender in complete status {}'.format(direction.capitalize(), tender['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_NOLOT_COMPLETE}, {"TENDER_ID": tender['id']})) yield tender else: logger.debug('{} sync: Skipping tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']), extra=journal_context(params={"TENDER_ID": tender['id']})) logger.info('Sleep {} sync...'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_SLEEP})) gevent.sleep(delay) logger.info('Restore {} sync'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_RESUME})) logger.debug('{} {}'.format(direction, params)) response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()}) def get_tender_contracts(self): while True: try: tender_to_sync = self.tenders_queue.get() tender = self.tenders_sync_client.get_tender(tender_to_sync['id'], extra_headers={'X-Client-Request-ID': generate_req_id()})['data'] db.put(tender_to_sync['id'], {'dateModified': tender_to_sync['dateModified']}) except Exception, e: logger.exception(e) logger.info('Put tender {} back to tenders queue'.format(tender_to_sync['id']), extra=journal_context(params={"TENDER_ID": tender_to_sync['id']})) self.tenders_queue.put(tender_to_sync) else: if 'contracts' not in tender: logger.warn('!!!No contracts found in tender {}'.format(tender['id']), extra=journal_context(params={"TENDER_ID": tender['id']})) continue for contract in tender['contracts']: if contract["status"] == "active": try: if not db.has(contract['id']): self.contracting_client.get_contract(contract['id']) else: logger.info('Contract {} exists in local db'.format(contract['id']), extra=journal_context(params={"CONTRACT_ID": contract['id']})) continue except ResourceNotFound: logger.info('Sync contract {} of tender {}'.format(contract['id'], tender['id']), extra=journal_context( {"MESSAGE_ID": DATABRIDGE_CONTRACT_TO_SYNC}, {"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']})) except Exception, e: logger.exception(e) logger.info('Put tender {} back to tenders queue'.format(tender_to_sync['id']), extra=journal_context(params={"TENDER_ID": tender_to_sync['id']})) self.tenders_queue.put(tender_to_sync) break else: db.put(contract['id'], True) logger.info('Contract exists {}'.format(contract['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_CONTRACT_EXISTS}, {"CONTRACT_ID": contract['id']})) continue contract['tender_id'] = tender['id'] contract['procuringEntity'] = tender['procuringEntity'] if not contract.get('items'): logger.info('Copying contract {} items'.format(contract['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_COPY_CONTRACT_ITEMS}, {"CONTRACT_ID": contract['id']})) if tender.get('lots'): related_awards = [aw for aw in tender['awards'] if aw['id'] == contract['awardID']] if related_awards: award = related_awards[0] if award.get("items"): logger.debug('Copying items from related award {}'.format(award['id'])) contract['items'] = award['items'] else: logger.debug('Copying items matching related lot {}'.format(award['relatedLot'])) contract['items'] = [item for item in tender['items'] if item['relatedLot'] == award['lotID']] else: logger.warn('Not found related award for contact {} of tender {}'.format(contract['id'], tender['id']), extra=journal_context(params={"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']})) else: logger.debug('Copying all tender {} items into contract {}'.format(tender['id'], contract['id']), extra=journal_context(params={"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']})) contract['items'] = tender['items'] if not contract.get('items'): logger.warn('Contact {} of tender {} does not contain items info'.format(contract['id'], tender['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_MISSING_CONTRACT_ITEMS}, {"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']})) self.handicap_contracts_queue.put(contract)
def __init__(self, config): super(ContractingDataBridge, self).__init__() self.config = config self.cache_db = Db(self.config.get('main')) self._backend = "redis" self._host = self.config.get('cache_host') self._port = self.config.get('cache_port') or 6379 self._db_name = self.config.get('cache_db_name') or 0 logger.info( "Caching backend: '{}', db name: '{}', host: '{}', port: '{}'". format(self.cache_db._backend, self.cache_db._db_name, self.cache_db._host, self.cache_db._port), extra=journal_context({"MESSAGE_ID": DATABRIDGE_INFO}, {})) self.on_error_delay = self.config_get('on_error_sleep_delay') or 5 self.jobs_watcher_delay = self.config_get('jobs_watcher_delay') or 15 queue_size = self.config_get('buffers_size') or 500 self.full_stack_sync_delay = self.config_get( 'full_stack_sync_delay') or 15 self.empty_stack_sync_delay = self.config_get( 'empty_stack_sync_delay') or 101 api_server = self.config_get('tenders_api_server') api_version = self.config_get('tenders_api_version') ro_api_server = self.config_get( 'public_tenders_api_server') or api_server contracting_api_server = self.config_get('contracting_api_server') contracting_api_version = self.config_get('contracting_api_version') self.tenders_sync_client = TendersClientSync( '', host_url=ro_api_server, api_version=api_version, ) self.client = TendersClient( self.config_get('api_token'), host_url=api_server, api_version=api_version, ) self.contracting_client = ContractingClient( self.config_get('api_token'), host_url=contracting_api_server, api_version=contracting_api_version) self.contracting_client_ro = self.contracting_client if self.config_get('public_tenders_api_server'): if api_server == contracting_api_server and api_version == contracting_api_version: self.contracting_client_ro = ContractingClient( self.config_get('api_token'), host_url=ro_api_server, api_version=api_version) self.initial_sync_point = {} self.initialization_event = gevent.event.Event() self.tenders_queue = Queue(maxsize=queue_size) self.handicap_contracts_queue = Queue(maxsize=queue_size) self.contracts_put_queue = Queue(maxsize=queue_size) self.contracts_retry_put_queue = Queue(maxsize=queue_size) self.basket = {}