class EdgeDataBridge(object):
"""Edge Bridge"""
def __init__(self, config):
super(EdgeDataBridge, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.retrievers_params = self.config_get('retrievers_params')
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version, key=''
)
self.couch_url = urljoin(
self.config_get('couch_url'),
self.config_get('public_db')
)
self.db = Database(self.couch_url,
session=Session(retry_delays=range(10)))
def config_get(self, name):
return self.config.get('main').get(name)
def get_teders_list(self):
for item in get_tenders(host=self.api_host, version=self.api_version,
key='', extra_params={'mode': '_all_'},
retrievers_params=self.retrievers_params):
yield (item["id"], item["dateModified"])
def save_tender_in_db(self, tender_id, date_modified):
tender_doc = self.db.get(tender_id)
if tender_doc:
if tender_doc['dateModified'] == date_modified:
return
tender = self.client.get_tender(tender_id).get('data')
if tender:
tender['_id'] = tender_id
tender['doc_type'] = 'Tender'
if tender_doc:
tender['_rev'] = tender_doc['_rev']
logger.info('Update tender {} '.format(tender_id))
else:
logger.info('Save tender {} '.format(tender_id))
try:
self.db.save(tender)
except Exception as e:
logger.info('Saving tender {} fail with error {}'.format(tender_id, e.message),
extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'})
else:
logger.info('Tender {} not found'.format(tender_id))
def run(self):
logger.info('Start Edge Bridge',
extra={'MESSAGE_ID': 'edge_bridge_start_bridge'})
logger.info('Start data sync...',
extra={'MESSAGE_ID': 'edge_bridge__data_sync'})
for tender_id, date_modified in self.get_teders_list():
self.save_tender_in_db(tender_id, date_modified)
class EdgeDataBridge(object):
"""Edge Bridge"""
def __init__(self, config):
super(EdgeDataBridge, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version,
key='')
self.couch_url = urljoin(self.config_get('couch_url'),
self.config_get('public_db'))
self.db = Database(self.couch_url,
session=Session(retry_delays=range(10)))
def config_get(self, name):
return self.config.get('main').get(name)
def get_teders_list(self):
for item in get_tenders(host=self.api_host,
version=self.api_version,
key='',
extra_params={'mode': '_all_'}):
yield (item["id"], item["dateModified"])
def save_tender_in_db(self, tender_id, date_modified):
tender_doc = self.db.get(tender_id)
if tender_doc:
if tender_doc['dateModified'] == date_modified:
return
tender = self.client.get_tender(tender_id).get('data')
if tender:
tender['_id'] = tender_id
tender['doc_type'] = 'Tender'
if tender_doc:
tender['_rev'] = tender_doc['_rev']
logger.info('Update tender {} '.format(tender_id))
else:
logger.info('Save tender {} '.format(tender_id))
try:
self.db.save(tender)
except Exception as e:
logger.info(
'Saving tender {} fail with error {}'.format(
tender_id, e.message),
extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'})
else:
logger.info('Tender {} not found'.format(tender_id))
def run(self):
logger.info('Start Edge Bridge',
extra={'MESSAGE_ID': 'edge_bridge_start_bridge'})
logger.info('Start data sync...',
extra={'MESSAGE_ID': 'edge_bridge__data_sync'})
for tender_id, date_modified in self.get_teders_list():
self.save_tender_in_db(tender_id, date_modified)
def get_tender(self, tender_id):
for i in range(5):
try:
resp = TendersClient.get_tender(self, tender_id)
except ResourceError as e:
logger.error("get_tender on %s error %s", tender_id, e)
time.sleep(10)
else:
return resp
raise ResourceError("Maximum retry reached")
def get_tender(self, tender_id):
for i in range(5):
try:
resp = TendersClient.get_tender(self, tender_id)
except ResourceError as e:
logger.error("get_tender on %s error %s", tender_id, e)
time.sleep(10)
else:
return resp
raise ResourceError("Maximum retry reached")
def get_tender(self, tender_id):
for i in range(5):
try:
if not self.headers.get('Cookie', None):
self.request_cookie()
return TendersClient.get_tender(self, tender_id)
except (socket.error, ResourceError) as e:
logger.error("get_tender %s reason %s", tender_id, str(e))
if i > 1:
self.headers.pop('Cookie', None)
#self.params.pop('offset', None)
time.sleep(10 * i + 10)
raise ResourceError("Maximum retry reached")
class ContractingDataBridge(object):
""" Contracting Data Bridge """
def __init__(self, config):
super(ContractingDataBridge, self).__init__()
self.config = config
self.contracting_client = ContractingClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version')
)
params = {'opt_fields': 'status,lots',
'mode': '_all_'}
self.tenders_client_forward = TendersClient(
'',
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
params=params
)
params_desc = deepcopy(params)
params_desc['descending'] = 1
self.tenders_client_backward = TendersClient(
'',
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
params=params_desc
)
self.client = TendersClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
self.tenders_queue = Queue()
self.handicap_contracts_queue = Queue()
self.contracts_put_queue = Queue()
self.contracts_retry_put_queue = Queue()
def config_get(self, name):
return self.config.get('main').get(name)
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
def get_tender_credentials(self, tender_id):
return self.client.extract_credentials(tender_id)
def get_tenders(self, client):
while True:
request_id = generate_req_id()
client.headers.update({'X-Client-Request-ID': request_id})
tenders_list = list(client.get_tenders())
delay = 101
if tenders_list:
delay = 15
logger.info("Client params: {}".format(client.params))
for tender in tenders_list:
if tender['status'] in ("active.qualification",
"active.awarded", "complete"):
if hasattr(tender, "lots"):
if any([1 for lot in tender['lots'] if lot['status'] == "complete"]):
logger.info('found multilot tender {} in status {}'.format(tender['id'], tender['status']))
yield tender
elif tender['status'] == "complete":
logger.info('Found tender in complete status {}'.format(tender['id']))
yield tender
else:
logger.debug('Skipping tender {} in status {}'.format(tender['id'], tender['status']))
logger.info('Sleep...')
time.sleep(delay)
def get_tender_contracts(self):
while True:
request_id = generate_req_id()
self.tenders_client_backward.headers.update({'X-Client-Request-ID': request_id})
tender = self.tenders_client_backward.get_tender(self.tenders_queue.get()['id'])['data']
if 'contracts' not in tender:
logger.warn('!!!No contracts found in tender {}'.format(tender['id']))
continue
for contract in tender['contracts']:
if contract["status"] == "active":
try:
self.contracting_client.get_contract(contract['id'])
except ResourceNotFound:
logger.info('Sync contract {} of tender {}'.format(contract['id'], tender['id']))
else:
logger.info('Contract exists {}'.format(contract['id']))
continue
contract['tender_id'] = tender['id']
contract['procuringEntity'] = tender['procuringEntity']
self.handicap_contracts_queue.put(contract)
def prepare_contract_data(self):
while True:
contract = self.handicap_contracts_queue.get()
try:
logger.info("Getting extra info for tender {}".format(contract['tender_id']))
tender_data = self.get_tender_credentials(contract['tender_id'])
except Exception, e:
logger.info("Can't get tender credentials {}".format(contract['tender_id']))
self.handicap_contracts_queue.put(contract)
else:
logger.debug("Got extra info for tender {}".format(
contract['tender_id']))
data = tender_data.data
if data.get('mode'):
contract['mode'] = data['mode']
contract['owner'] = data['owner']
contract['tender_token'] = data['tender_token']
del contract['status'] # create contract in 'draft' status
self.contracts_put_queue.put(contract)
gevent.sleep(0)
class Oracle(object):
def __init__(self, config):
super(Oracle, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.retrievers_params = self.config_get('retrievers_params')
self.db = DB
try:
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version,
key='')
except MissingSchema:
raise ConfigError(
'In config dictionary empty or missing \'tenders_api_server\'')
except ConnectionError as e:
raise e
def config_get(self, name):
try:
return self.config.get('main').get(name)
except AttributeError as e:
raise ConfigError('In config dictionary missed section \'main\'')
def get_teders_list(self):
for item in get_tenders(host=self.api_host,
version=self.api_version,
key='',
extra_params={
'mode': '_all_',
'opt_fields': 'status'
},
retrievers_params=self.retrievers_params):
yield (item["id"], item["status"])
def approve_bids(self, tender_id, date_modified):
# TODO: At this point must be get_bids
if self.db[tender['id']] == tender['status']:
continue
tender = self.client.get_tender(tender_id).get('data')
if not tender:
logger.info('Tender {} not found'.format(tender_id))
continue
for bid in tender['bids']:
doc_id = "{}_{}".format(tender['id'], bid['id'])
if self.db[doc_id] == tender['status']:
continue
shard_key = "Oracle approve: " + bid['key']
self.client.patch_bid(tender['id'], bid['id'],
{'data': {
"shard": shard_key
}})
self.db[doc_id] = tender['status']
self.db[tender['id']] = tender['status']
def run(self):
logger.info('Start Oracle', extra={'MESSAGE_ID': 'start_oracle'})
logger.info('Start data sync...',
extra={'MESSAGE_ID': 'oracle__data_sync'})
for tender_id, status in self.get_teders_list():
self.aporove_bids(tender_id, date_modified)
class EdgeDataBridge(object):
"""Edge Bridge"""
def __init__(self, config):
super(EdgeDataBridge, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.retrievers_params = self.config_get('retrievers_params')
try:
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version,
key='')
except MissingSchema:
raise DataBridgeConfigError(
'In config dictionary empty or missing \'tenders_api_server\'')
except ConnectionError as e:
raise e
self.couch_url = urljoin(self.config_get('couch_url'),
self.config_get('public_db'))
self.db = Database(self.couch_url,
session=Session(retry_delays=range(10)))
try:
self.db.info()
except ResourceNotFound:
error_message = "Database with name '" + self.config_get(
'public_db') + "' doesn\'t exist"
raise DataBridgeConfigError(error_message)
except error as e:
if e.errno == errno.ECONNREFUSED:
raise DataBridgeConfigError(
"Connection refused: 'couch_url' is invalid in config dictionary"
)
except AttributeError as e:
raise DataBridgeConfigError(
'\'couch_url\' is missed or empty in config dictionary.')
except KeyError as e:
if e.message == 'db_name':
raise DataBridgeConfigError(
'\'public_db\' name is missed or empty in config dictionary'
)
def config_get(self, name):
try:
return self.config.get('main').get(name)
except AttributeError as e:
raise DataBridgeConfigError(
'In config dictionary missed section \'main\'')
def get_teders_list(self):
for item in get_tenders(host=self.api_host,
version=self.api_version,
key='',
extra_params={'mode': '_all_'},
retrievers_params=self.retrievers_params):
yield (item["id"], item["dateModified"])
def save_tender_in_db(self, tender_id, date_modified):
tender_doc = self.db.get(tender_id)
if tender_doc:
if tender_doc['dateModified'] == date_modified:
return
tender = self.client.get_tender(tender_id).get('data')
if tender:
tender['_id'] = tender_id
tender['doc_type'] = 'Tender'
if tender_doc:
tender['_rev'] = tender_doc['_rev']
logger.info('Update tender {} '.format(tender_id))
else:
logger.info('Save tender {} '.format(tender_id))
try:
self.db.save(tender)
except Exception as e:
logger.info(
'Saving tender {} fail with error {}'.format(
tender_id, e.message),
extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'})
else:
logger.info('Tender {} not found'.format(tender_id))
def run(self):
logger.info('Start Edge Bridge',
extra={'MESSAGE_ID': 'edge_bridge_start_bridge'})
logger.info('Start data sync...',
extra={'MESSAGE_ID': 'edge_bridge__data_sync'})
for tender_id, date_modified in self.get_teders_list():
self.save_tender_in_db(tender_id, date_modified)