def __init__(self, key, config):
params = dict()
if config['mode'] in ('test', '_all_'):
params['mode'] = config['mode']
TendersClient.__init__(self, key, config['host_url'], config['api_version'], params)
if config.get('resource'):
self.prefix_path = '/api/{}/{}'.format(config['api_version'], config['resource'])
if config['timeout']:
socket.setdefaulttimeout(float(config['timeout']))
def __init__(self, key, config):
params = {'limit': 1000}
if config['mode'] in ('test', '_all_'):
params['mode'] = config['mode']
if config['timeout']:
socket.setdefaulttimeout(float(config['timeout']))
TendersClient.__init__(self, key, config['host_url'], config['api_version'], params)
if config.get('resource'):
self.prefix_path = '/api/{}/{}'.format(config['api_version'], config['resource'])
self.allow_preload = config.get('preload', None)
self.api_version = config['api_version']
def __init__(self, key, config):
params = dict()
if config['mode'] in ('test', '_all_'):
params['mode'] = config['mode']
TendersClient.__init__(self, key, config['host_url'],
config['api_version'], params)
if config.get('resource'):
self.prefix_path = '/api/{}/{}'.format(config['api_version'],
config['resource'])
if config['timeout']:
socket.setdefaulttimeout(float(config['timeout']))
def clients_initialize(self):
self.client = TendersClient(
self.config_get('api_token'),
host_url=self.api_server, api_version=self.api_version,
)
self.contracting_client_init()
self.tenders_sync_client = TendersClientSync('',
host_url=self.ro_api_server, api_version=self.api_version,
)
def __init__(self, config):
super(EdgeDataBridge, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version,
key='')
self.couch_url = urljoin(self.config_get('couch_url'),
self.config_get('public_db'))
self.db = Database(self.couch_url,
session=Session(retry_delays=range(10)))
def __init__(self, config):
super(ContractingDataBridge, self).__init__()
self.config = config
self.tenders_sync_client = TendersClientSync('',
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
self.client = TendersClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
self.contracting_client = ContractingClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version')
)
self.initial_sync_point = {}
self.tenders_queue = Queue(maxsize=500)
self.handicap_contracts_queue = Queue(maxsize=500)
self.contracts_put_queue = Queue(maxsize=500)
self.contracts_retry_put_queue = Queue(maxsize=500)
class EdgeDataBridge(object):
"""Edge Bridge"""
def __init__(self, config):
super(EdgeDataBridge, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.retrievers_params = self.config_get('retrievers_params')
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version, key=''
)
self.couch_url = urljoin(
self.config_get('couch_url'),
self.config_get('public_db')
)
self.db = Database(self.couch_url,
session=Session(retry_delays=range(10)))
def config_get(self, name):
return self.config.get('main').get(name)
def get_teders_list(self):
for item in get_tenders(host=self.api_host, version=self.api_version,
key='', extra_params={'mode': '_all_'},
retrievers_params=self.retrievers_params):
yield (item["id"], item["dateModified"])
def save_tender_in_db(self, tender_id, date_modified):
tender_doc = self.db.get(tender_id)
if tender_doc:
if tender_doc['dateModified'] == date_modified:
return
tender = self.client.get_tender(tender_id).get('data')
if tender:
tender['_id'] = tender_id
tender['doc_type'] = 'Tender'
if tender_doc:
tender['_rev'] = tender_doc['_rev']
logger.info('Update tender {} '.format(tender_id))
else:
logger.info('Save tender {} '.format(tender_id))
try:
self.db.save(tender)
except Exception as e:
logger.info('Saving tender {} fail with error {}'.format(tender_id, e.message),
extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'})
else:
logger.info('Tender {} not found'.format(tender_id))
def run(self):
logger.info('Start Edge Bridge',
extra={'MESSAGE_ID': 'edge_bridge_start_bridge'})
logger.info('Start data sync...',
extra={'MESSAGE_ID': 'edge_bridge__data_sync'})
for tender_id, date_modified in self.get_teders_list():
self.save_tender_in_db(tender_id, date_modified)
class EdgeDataBridge(object):
"""Edge Bridge"""
def __init__(self, config):
super(EdgeDataBridge, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version,
key='')
self.couch_url = urljoin(self.config_get('couch_url'),
self.config_get('public_db'))
self.db = Database(self.couch_url,
session=Session(retry_delays=range(10)))
def config_get(self, name):
return self.config.get('main').get(name)
def get_teders_list(self):
for item in get_tenders(host=self.api_host,
version=self.api_version,
key='',
extra_params={'mode': '_all_'}):
yield (item["id"], item["dateModified"])
def save_tender_in_db(self, tender_id, date_modified):
tender_doc = self.db.get(tender_id)
if tender_doc:
if tender_doc['dateModified'] == date_modified:
return
tender = self.client.get_tender(tender_id).get('data')
if tender:
tender['_id'] = tender_id
tender['doc_type'] = 'Tender'
if tender_doc:
tender['_rev'] = tender_doc['_rev']
logger.info('Update tender {} '.format(tender_id))
else:
logger.info('Save tender {} '.format(tender_id))
try:
self.db.save(tender)
except Exception as e:
logger.info(
'Saving tender {} fail with error {}'.format(
tender_id, e.message),
extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'})
else:
logger.info('Tender {} not found'.format(tender_id))
def run(self):
logger.info('Start Edge Bridge',
extra={'MESSAGE_ID': 'edge_bridge_start_bridge'})
logger.info('Start data sync...',
extra={'MESSAGE_ID': 'edge_bridge__data_sync'})
for tender_id, date_modified in self.get_teders_list():
self.save_tender_in_db(tender_id, date_modified)
def __init__(self, config):
super(Oracle, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.retrievers_params = self.config_get('retrievers_params')
self.db = DB
try:
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version,
key='')
except MissingSchema:
raise ConfigError(
'In config dictionary empty or missing \'tenders_api_server\'')
except ConnectionError as e:
raise e
def get_tender(self, tender_id):
for i in range(5):
try:
resp = TendersClient.get_tender(self, tender_id)
except ResourceError as e:
logger.error("get_tender on %s error %s", tender_id, e)
time.sleep(10)
else:
return resp
raise ResourceError("Maximum retry reached")
def get_tender(self, tender_id):
for i in range(5):
try:
resp = TendersClient.get_tender(self, tender_id)
except ResourceError as e:
logger.error("get_tender on %s error %s", tender_id, e)
time.sleep(10)
else:
return resp
raise ResourceError("Maximum retry reached")
def __init__(self, config):
super(EdgeDataBridge, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.retrievers_params = self.config_get('retrievers_params')
try:
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version,
key='')
except MissingSchema:
raise DataBridgeConfigError(
'In config dictionary empty or missing \'tenders_api_server\'')
except ConnectionError as e:
raise e
self.couch_url = urljoin(self.config_get('couch_url'),
self.config_get('public_db'))
self.db = Database(self.couch_url,
session=Session(retry_delays=range(10)))
try:
self.db.info()
except ResourceNotFound:
error_message = "Database with name '" + self.config_get(
'public_db') + "' doesn\'t exist"
raise DataBridgeConfigError(error_message)
except error as e:
if e.errno == errno.ECONNREFUSED:
raise DataBridgeConfigError(
"Connection refused: 'couch_url' is invalid in config dictionary"
)
except AttributeError as e:
raise DataBridgeConfigError(
'\'couch_url\' is missed or empty in config dictionary.')
except KeyError as e:
if e.message == 'db_name':
raise DataBridgeConfigError(
'\'public_db\' name is missed or empty in config dictionary'
)
def get_tender(self, tender_id):
for i in range(5):
try:
if not self.headers.get('Cookie', None):
self.request_cookie()
return TendersClient.get_tender(self, tender_id)
except (socket.error, ResourceError) as e:
logger.error("get_tender %s reason %s", tender_id, str(e))
if i > 1:
self.headers.pop('Cookie', None)
#self.params.pop('offset', None)
time.sleep(10 * i + 10)
raise ResourceError("Maximum retry reached")
def __init__(self, config):
super(ContractingDataBridge, self).__init__()
self.config = config
self.contracting_client = ContractingClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version')
)
params = {'opt_fields': 'status,lots',
'mode': '_all_'}
self.tenders_client_forward = TendersClient(
'',
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
params=params
)
params_desc = deepcopy(params)
params_desc['descending'] = 1
self.tenders_client_backward = TendersClient(
'',
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
params=params_desc
)
self.client = TendersClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
self.tenders_queue = Queue()
self.handicap_contracts_queue = Queue()
self.contracts_put_queue = Queue()
self.contracts_retry_put_queue = Queue()
def __init__(self, config):
super(EdgeDataBridge, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.retrievers_params = self.config_get('retrievers_params')
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version, key=''
)
self.couch_url = urljoin(
self.config_get('couch_url'),
self.config_get('public_db')
)
self.db = Database(self.couch_url,
session=Session(retry_delays=range(10)))
def validate_credentials_generate(request):
try:
token = get_access_token(request)
except ValueError:
raise_operation_error(request, 'No access token was provided.')
try:
response = TendersClient(
request.registry.api_token,
host_url=request.registry.api_server,
api_version=request.registry.api_version,
).extract_credentials(request.validated['monitoring'].tender_id)
except ResourceNotFound:
raise_operation_error(
request, 'Tender {} not found'.format(
request.validated['monitoring'].tender_id))
else:
if sha512(token).hexdigest() != response['data']['tender_token']:
raise forbidden(request)
class ContractingDataBridge(object):
""" Contracting Data Bridge """
def __init__(self, config):
super(ContractingDataBridge, self).__init__()
self.config = config
self.cache_db = Db(self.config.get('main'))
self._backend = "redis"
self._host = self.config.get('cache_host')
self._port = self.config.get('cache_port') or 6379
self._db_name = self.config.get('cache_db_name') or 0
logger.info("Caching backend: '{}', db name: '{}', host: '{}', port: '{}'".format(self.cache_db._backend,
self.cache_db._db_name,
self.cache_db._host,
self.cache_db._port),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_INFO}, {}))
self.on_error_delay = self.config_get('on_error_sleep_delay') or 5
self.jobs_watcher_delay = self.config_get('jobs_watcher_delay') or 15
queue_size = self.config_get('buffers_size') or 500
self.full_stack_sync_delay = self.config_get('full_stack_sync_delay') or 15
self.empty_stack_sync_delay = self.config_get('empty_stack_sync_delay') or 101
self.api_server = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.ro_api_server = self.config_get('public_tenders_api_server') or self.api_server
self.contracting_api_server = self.config_get('contracting_api_server')
self.contracting_api_version = self.config_get('contracting_api_version')
self.clients_initialize()
self.initial_sync_point = {}
self.initialization_event = gevent.event.Event()
self.tenders_queue = Queue(maxsize=queue_size)
self.handicap_contracts_queue = Queue(maxsize=queue_size)
self.contracts_put_queue = Queue(maxsize=queue_size)
self.contracts_retry_put_queue = Queue(maxsize=queue_size)
self.basket = {}
def contracting_client_init(self):
logger.info('Initialization contracting clients.', extra=journal_context({"MESSAGE_ID": DATABRIDGE_INFO}, {}))
self.contracting_client = ContractingClient(
self.config_get('api_token'),
host_url=self.contracting_api_server, api_version=self.contracting_api_version
)
self.contracting_client_ro = self.contracting_client
if self.config_get('public_tenders_api_server'):
if self.api_server == self.contracting_api_server and self.api_version == self.contracting_api_version:
self.contracting_client_ro = ContractingClient(
'',
host_url=self.ro_api_server, api_version=self.api_version
)
def clients_initialize(self):
self.client = TendersClient(
self.config_get('api_token'),
host_url=self.api_server, api_version=self.api_version,
)
self.contracting_client_init()
self.tenders_sync_client = TendersClientSync('',
host_url=self.ro_api_server, api_version=self.api_version,
)
def config_get(self, name):
return self.config.get('main').get(name)
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
def get_tender_credentials(self, tender_id):
self.client.headers.update({'X-Client-Request-ID': generate_req_id()})
logger.info("Getting credentials for tender {}".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GET_CREDENTIALS},
{"TENDER_ID": tender_id}))
data = self.client.extract_credentials(tender_id)
logger.info("Got tender {} credentials".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GOT_CREDENTIALS},
{"TENDER_ID": tender_id}))
return data
def initialize_sync(self, params=None, direction=None):
self.initialization_event.clear()
if direction == "backward":
assert params['descending']
response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()})
# set values in reverse order due to 'descending' option
self.initial_sync_point = {'forward_offset': response.prev_page.offset,
'backward_offset': response.next_page.offset}
self.initialization_event.set() # wake up forward worker
logger.info("Initial sync point {}".format(self.initial_sync_point))
return response
else:
assert 'descending' not in params
gevent.wait([self.initialization_event])
params['offset'] = self.initial_sync_point['forward_offset']
logger.info("Starting forward sync from offset {}".format(params['offset']))
return self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()})
def get_tenders(self, params={}, direction=""):
response = self.initialize_sync(params=params, direction=direction)
while not (params.get('descending') and not len(response.data) and params.get('offset') == response.next_page.offset):
tenders_list = response.data
params['offset'] = response.next_page.offset
delay = self.empty_stack_sync_delay
if tenders_list:
delay = self.full_stack_sync_delay
logger.info("Client {} params: {}".format(direction, params))
for tender in tenders_list:
if tender.get('procurementMethodType') in ['competitiveDialogueUA', 'competitiveDialogueEU']:
logger.info('Skipping {} tender {}'.format(tender['procurementMethodType'], tender['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_INFO}, params={"TENDER_ID": tender['id']}))
continue
if tender['status'] in ("active.qualification", "active",
"active.awarded", "complete"):
if hasattr(tender, "lots"):
if any([1 for lot in tender['lots'] if lot['status'] == "complete"]):
logger.info('{} sync: Found multilot tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_MULTILOT_COMPLETE}, {"TENDER_ID": tender['id']}))
yield tender
elif tender['status'] == "complete":
logger.info('{} sync: Found tender in complete status {}'.format(direction.capitalize(), tender['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_NOLOT_COMPLETE}, {"TENDER_ID": tender['id']}))
yield tender
else:
logger.debug('{} sync: Skipping tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']),
extra=journal_context(params={"TENDER_ID": tender['id']}))
logger.info('Sleep {} sync...'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_SLEEP}))
gevent.sleep(delay)
logger.info('Restore {} sync'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_RESUME}))
logger.debug('{} {}'.format(direction, params))
response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()})
def _put_tender_in_cache_by_contract(self, contract, tender_id):
dateModified = self.basket.get(contract['id'])
if dateModified:
# TODO: save tender in cache only if all active contracts are
# handled successfully
self.cache_db.put(tender_id, dateModified)
self.basket.pop(contract['id'], None)
def _get_tender_contracts(self):
try:
tender_to_sync = self.tenders_queue.get()
tender = self.tenders_sync_client.get_tender(tender_to_sync['id'],
extra_headers={'X-Client-Request-ID': generate_req_id()})['data']
except Exception, e:
logger.warn('Fail to get tender info {}'.format(tender_to_sync['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_EXCEPTION}, params={"TENDER_ID": tender_to_sync['id']}))
logger.exception(e)
logger.info('Put tender {} back to tenders queue'.format(tender_to_sync['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_EXCEPTION}, params={"TENDER_ID": tender_to_sync['id']}))
self.tenders_queue.put(tender_to_sync)
gevent.sleep(self.on_error_delay)
else:
class EdgeDataBridge(object):
"""Edge Bridge"""
def __init__(self, config):
super(EdgeDataBridge, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.retrievers_params = self.config_get('retrievers_params')
try:
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version,
key='')
except MissingSchema:
raise DataBridgeConfigError(
'In config dictionary empty or missing \'tenders_api_server\'')
except ConnectionError as e:
raise e
self.couch_url = urljoin(self.config_get('couch_url'),
self.config_get('public_db'))
self.db = Database(self.couch_url,
session=Session(retry_delays=range(10)))
try:
self.db.info()
except ResourceNotFound:
error_message = "Database with name '" + self.config_get(
'public_db') + "' doesn\'t exist"
raise DataBridgeConfigError(error_message)
except error as e:
if e.errno == errno.ECONNREFUSED:
raise DataBridgeConfigError(
"Connection refused: 'couch_url' is invalid in config dictionary"
)
except AttributeError as e:
raise DataBridgeConfigError(
'\'couch_url\' is missed or empty in config dictionary.')
except KeyError as e:
if e.message == 'db_name':
raise DataBridgeConfigError(
'\'public_db\' name is missed or empty in config dictionary'
)
def config_get(self, name):
try:
return self.config.get('main').get(name)
except AttributeError as e:
raise DataBridgeConfigError(
'In config dictionary missed section \'main\'')
def get_teders_list(self):
for item in get_tenders(host=self.api_host,
version=self.api_version,
key='',
extra_params={'mode': '_all_'},
retrievers_params=self.retrievers_params):
yield (item["id"], item["dateModified"])
def save_tender_in_db(self, tender_id, date_modified):
tender_doc = self.db.get(tender_id)
if tender_doc:
if tender_doc['dateModified'] == date_modified:
return
tender = self.client.get_tender(tender_id).get('data')
if tender:
tender['_id'] = tender_id
tender['doc_type'] = 'Tender'
if tender_doc:
tender['_rev'] = tender_doc['_rev']
logger.info('Update tender {} '.format(tender_id))
else:
logger.info('Save tender {} '.format(tender_id))
try:
self.db.save(tender)
except Exception as e:
logger.info(
'Saving tender {} fail with error {}'.format(
tender_id, e.message),
extra={'MESSAGE_ID': 'edge_bridge_fail_save_in_db'})
else:
logger.info('Tender {} not found'.format(tender_id))
def run(self):
logger.info('Start Edge Bridge',
extra={'MESSAGE_ID': 'edge_bridge_start_bridge'})
logger.info('Start data sync...',
extra={'MESSAGE_ID': 'edge_bridge__data_sync'})
for tender_id, date_modified in self.get_teders_list():
self.save_tender_in_db(tender_id, date_modified)
class Oracle(object):
def __init__(self, config):
super(Oracle, self).__init__()
self.config = config
self.api_host = self.config_get('tenders_api_server')
self.api_version = self.config_get('tenders_api_version')
self.retrievers_params = self.config_get('retrievers_params')
self.db = DB
try:
self.client = TendersClient(host_url=self.api_host,
api_version=self.api_version,
key='')
except MissingSchema:
raise ConfigError(
'In config dictionary empty or missing \'tenders_api_server\'')
except ConnectionError as e:
raise e
def config_get(self, name):
try:
return self.config.get('main').get(name)
except AttributeError as e:
raise ConfigError('In config dictionary missed section \'main\'')
def get_teders_list(self):
for item in get_tenders(host=self.api_host,
version=self.api_version,
key='',
extra_params={
'mode': '_all_',
'opt_fields': 'status'
},
retrievers_params=self.retrievers_params):
yield (item["id"], item["status"])
def approve_bids(self, tender_id, date_modified):
# TODO: At this point must be get_bids
if self.db[tender['id']] == tender['status']:
continue
tender = self.client.get_tender(tender_id).get('data')
if not tender:
logger.info('Tender {} not found'.format(tender_id))
continue
for bid in tender['bids']:
doc_id = "{}_{}".format(tender['id'], bid['id'])
if self.db[doc_id] == tender['status']:
continue
shard_key = "Oracle approve: " + bid['key']
self.client.patch_bid(tender['id'], bid['id'],
{'data': {
"shard": shard_key
}})
self.db[doc_id] = tender['status']
self.db[tender['id']] = tender['status']
def run(self):
logger.info('Start Oracle', extra={'MESSAGE_ID': 'start_oracle'})
logger.info('Start data sync...',
extra={'MESSAGE_ID': 'oracle__data_sync'})
for tender_id, status in self.get_teders_list():
self.aporove_bids(tender_id, date_modified)
class ContractingDataBridge(object):
""" Contracting Data Bridge """
def __init__(self, config):
super(ContractingDataBridge, self).__init__()
self.config = config
self.contracting_client = ContractingClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version')
)
params = {'opt_fields': 'status,lots',
'mode': '_all_'}
self.tenders_client_forward = TendersClient(
'',
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
params=params
)
params_desc = deepcopy(params)
params_desc['descending'] = 1
self.tenders_client_backward = TendersClient(
'',
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
params=params_desc
)
self.client = TendersClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
self.tenders_queue = Queue()
self.handicap_contracts_queue = Queue()
self.contracts_put_queue = Queue()
self.contracts_retry_put_queue = Queue()
def config_get(self, name):
return self.config.get('main').get(name)
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
def get_tender_credentials(self, tender_id):
return self.client.extract_credentials(tender_id)
def get_tenders(self, client):
while True:
request_id = generate_req_id()
client.headers.update({'X-Client-Request-ID': request_id})
tenders_list = list(client.get_tenders())
delay = 101
if tenders_list:
delay = 15
logger.info("Client params: {}".format(client.params))
for tender in tenders_list:
if tender['status'] in ("active.qualification",
"active.awarded", "complete"):
if hasattr(tender, "lots"):
if any([1 for lot in tender['lots'] if lot['status'] == "complete"]):
logger.info('found multilot tender {} in status {}'.format(tender['id'], tender['status']))
yield tender
elif tender['status'] == "complete":
logger.info('Found tender in complete status {}'.format(tender['id']))
yield tender
else:
logger.debug('Skipping tender {} in status {}'.format(tender['id'], tender['status']))
logger.info('Sleep...')
time.sleep(delay)
def get_tender_contracts(self):
while True:
request_id = generate_req_id()
self.tenders_client_backward.headers.update({'X-Client-Request-ID': request_id})
tender = self.tenders_client_backward.get_tender(self.tenders_queue.get()['id'])['data']
if 'contracts' not in tender:
logger.warn('!!!No contracts found in tender {}'.format(tender['id']))
continue
for contract in tender['contracts']:
if contract["status"] == "active":
try:
self.contracting_client.get_contract(contract['id'])
except ResourceNotFound:
logger.info('Sync contract {} of tender {}'.format(contract['id'], tender['id']))
else:
logger.info('Contract exists {}'.format(contract['id']))
continue
contract['tender_id'] = tender['id']
contract['procuringEntity'] = tender['procuringEntity']
self.handicap_contracts_queue.put(contract)
def prepare_contract_data(self):
while True:
contract = self.handicap_contracts_queue.get()
try:
logger.info("Getting extra info for tender {}".format(contract['tender_id']))
tender_data = self.get_tender_credentials(contract['tender_id'])
except Exception, e:
logger.info("Can't get tender credentials {}".format(contract['tender_id']))
self.handicap_contracts_queue.put(contract)
else:
logger.debug("Got extra info for tender {}".format(
contract['tender_id']))
data = tender_data.data
if data.get('mode'):
contract['mode'] = data['mode']
contract['owner'] = data['owner']
contract['tender_token'] = data['tender_token']
del contract['status'] # create contract in 'draft' status
self.contracts_put_queue.put(contract)
gevent.sleep(0)
class ContractingDataBridge(object):
""" Contracting Data Bridge """
def __init__(self, config):
super(ContractingDataBridge, self).__init__()
self.config = config
self.tenders_sync_client = TendersClientSync('',
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
self.client = TendersClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
self.contracting_client = ContractingClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version')
)
self.initial_sync_point = {}
self.tenders_queue = Queue(maxsize=500)
self.handicap_contracts_queue = Queue(maxsize=500)
self.contracts_put_queue = Queue(maxsize=500)
self.contracts_retry_put_queue = Queue(maxsize=500)
def config_get(self, name):
return self.config.get('main').get(name)
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
def get_tender_credentials(self, tender_id):
self.client.headers.update({'X-Client-Request-ID': generate_req_id()})
logger.info("Getting credentials for tender {}".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GET_CREDENTIALS},
{"TENDER_ID": tender_id}))
data = self.client.extract_credentials(tender_id)
logger.info("Got tender {} credentials".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GOT_CREDENTIALS},
{"TENDER_ID": tender_id}))
return data
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
def initialize_sync(self, params=None, direction=None):
# TODO use gevent.Event to wake up forward sync instead of checking
# initial sync point
if direction == "backward":
assert params['descending']
response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()})
# set values in reverse order due to 'descending' option
self.initial_sync_point = {'forward_offset': response.prev_page.offset,
'backward_offset': response.next_page.offset}
logger.info("Initial sync point {}".format(self.initial_sync_point))
return response
elif not self.initial_sync_point:
raise ValueError
else:
assert 'descending' not in params
params['offset'] = self.initial_sync_point['forward_offset']
logger.info("Starting forward sync from offset {}".format(params['offset']))
return self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()})
def get_tenders(self, params={}, direction=""):
response = self.initialize_sync(params=params, direction=direction)
while not (params.get('descending') and not len(response.data) and params.get('offset') == response.next_page.offset):
tenders_list = response.data
params['offset'] = response.next_page.offset
delay = 101
if tenders_list:
delay = 15
logger.info("Client {} params: {}".format(direction, params))
for tender in tenders_list:
if tender['status'] in ("active.qualification",
"active.awarded", "complete"):
if hasattr(tender, "lots"):
if any([1 for lot in tender['lots'] if lot['status'] == "complete"]):
logger.info('{} sync: Found multilot tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_MULTILOT_COMPLETE}, {"TENDER_ID": tender['id']}))
yield tender
elif tender['status'] == "complete":
logger.info('{} sync: Found tender in complete status {}'.format(direction.capitalize(), tender['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_NOLOT_COMPLETE}, {"TENDER_ID": tender['id']}))
yield tender
else:
logger.debug('{} sync: Skipping tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']),
extra=journal_context(params={"TENDER_ID": tender['id']}))
logger.info('Sleep {} sync...'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_SLEEP}))
gevent.sleep(delay)
logger.info('Restore {} sync'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_RESUME}))
logger.debug('{} {}'.format(direction, params))
response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()})
def get_tender_contracts(self):
while True:
try:
tender_to_sync = self.tenders_queue.get()
tender = self.tenders_sync_client.get_tender(tender_to_sync['id'],
extra_headers={'X-Client-Request-ID': generate_req_id()})['data']
db.put(tender_to_sync['id'], {'dateModified': tender_to_sync['dateModified']})
except Exception, e:
logger.exception(e)
logger.info('Put tender {} back to tenders queue'.format(tender_to_sync['id']), extra=journal_context(params={"TENDER_ID": tender_to_sync['id']}))
self.tenders_queue.put(tender_to_sync)
else:
if 'contracts' not in tender:
logger.warn('!!!No contracts found in tender {}'.format(tender['id']), extra=journal_context(params={"TENDER_ID": tender['id']}))
continue
for contract in tender['contracts']:
if contract["status"] == "active":
try:
if not db.has(contract['id']):
self.contracting_client.get_contract(contract['id'])
else:
logger.info('Contract {} exists in local db'.format(contract['id']), extra=journal_context(params={"CONTRACT_ID": contract['id']}))
continue
except ResourceNotFound:
logger.info('Sync contract {} of tender {}'.format(contract['id'], tender['id']), extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_CONTRACT_TO_SYNC}, {"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']}))
except Exception, e:
logger.exception(e)
logger.info('Put tender {} back to tenders queue'.format(tender_to_sync['id']), extra=journal_context(params={"TENDER_ID": tender_to_sync['id']}))
self.tenders_queue.put(tender_to_sync)
break
else:
db.put(contract['id'], True)
logger.info('Contract exists {}'.format(contract['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_CONTRACT_EXISTS},
{"CONTRACT_ID": contract['id']}))
continue
contract['tender_id'] = tender['id']
contract['procuringEntity'] = tender['procuringEntity']
if not contract.get('items'):
logger.info('Copying contract {} items'.format(contract['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_COPY_CONTRACT_ITEMS},
{"CONTRACT_ID": contract['id']}))
if tender.get('lots'):
related_awards = [aw for aw in tender['awards'] if aw['id'] == contract['awardID']]
if related_awards:
award = related_awards[0]
if award.get("items"):
logger.debug('Copying items from related award {}'.format(award['id']))
contract['items'] = award['items']
else:
logger.debug('Copying items matching related lot {}'.format(award['relatedLot']))
contract['items'] = [item for item in tender['items'] if item['relatedLot'] == award['lotID']]
else:
logger.warn('Not found related award for contact {} of tender {}'.format(contract['id'], tender['id']),
extra=journal_context(params={"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']}))
else:
logger.debug('Copying all tender {} items into contract {}'.format(tender['id'], contract['id']),
extra=journal_context(params={"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']}))
contract['items'] = tender['items']
if not contract.get('items'):
logger.warn('Contact {} of tender {} does not contain items info'.format(contract['id'], tender['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_MISSING_CONTRACT_ITEMS},
{"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']}))
self.handicap_contracts_queue.put(contract)
def __init__(self, config):
super(ContractingDataBridge, self).__init__()
self.config = config
self.cache_db = Db(self.config.get('main'))
self._backend = "redis"
self._host = self.config.get('cache_host')
self._port = self.config.get('cache_port') or 6379
self._db_name = self.config.get('cache_db_name') or 0
logger.info(
"Caching backend: '{}', db name: '{}', host: '{}', port: '{}'".
format(self.cache_db._backend, self.cache_db._db_name,
self.cache_db._host, self.cache_db._port),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_INFO}, {}))
self.on_error_delay = self.config_get('on_error_sleep_delay') or 5
self.jobs_watcher_delay = self.config_get('jobs_watcher_delay') or 15
queue_size = self.config_get('buffers_size') or 500
self.full_stack_sync_delay = self.config_get(
'full_stack_sync_delay') or 15
self.empty_stack_sync_delay = self.config_get(
'empty_stack_sync_delay') or 101
api_server = self.config_get('tenders_api_server')
api_version = self.config_get('tenders_api_version')
ro_api_server = self.config_get(
'public_tenders_api_server') or api_server
contracting_api_server = self.config_get('contracting_api_server')
contracting_api_version = self.config_get('contracting_api_version')
self.tenders_sync_client = TendersClientSync(
'',
host_url=ro_api_server,
api_version=api_version,
)
self.client = TendersClient(
self.config_get('api_token'),
host_url=api_server,
api_version=api_version,
)
self.contracting_client = ContractingClient(
self.config_get('api_token'),
host_url=contracting_api_server,
api_version=contracting_api_version)
self.contracting_client_ro = self.contracting_client
if self.config_get('public_tenders_api_server'):
if api_server == contracting_api_server and api_version == contracting_api_version:
self.contracting_client_ro = ContractingClient(
self.config_get('api_token'),
host_url=ro_api_server,
api_version=api_version)
self.initial_sync_point = {}
self.initialization_event = gevent.event.Event()
self.tenders_queue = Queue(maxsize=queue_size)
self.handicap_contracts_queue = Queue(maxsize=queue_size)
self.contracts_put_queue = Queue(maxsize=queue_size)
self.contracts_retry_put_queue = Queue(maxsize=queue_size)
self.basket = {}
