class ContractingDataBridge(object):
""" Contracting Data Bridge """
def __init__(self, config):
super(ContractingDataBridge, self).__init__()
self.config = config
self.tenders_sync_client = TendersClientSync('',
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
self.client = TendersClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
self.contracting_client = ContractingClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version')
)
self.initial_sync_point = {}
self.tenders_queue = Queue(maxsize=500)
self.handicap_contracts_queue = Queue(maxsize=500)
self.contracts_put_queue = Queue(maxsize=500)
self.contracts_retry_put_queue = Queue(maxsize=500)
def config_get(self, name):
return self.config.get('main').get(name)
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
def get_tender_credentials(self, tender_id):
self.client.headers.update({'X-Client-Request-ID': generate_req_id()})
logger.info("Getting credentials for tender {}".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GET_CREDENTIALS},
{"TENDER_ID": tender_id}))
data = self.client.extract_credentials(tender_id)
logger.info("Got tender {} credentials".format(tender_id), extra=journal_context({"MESSAGE_ID": DATABRIDGE_GOT_CREDENTIALS},
{"TENDER_ID": tender_id}))
return data
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
def initialize_sync(self, params=None, direction=None):
# TODO use gevent.Event to wake up forward sync instead of checking
# initial sync point
if direction == "backward":
assert params['descending']
response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()})
# set values in reverse order due to 'descending' option
self.initial_sync_point = {'forward_offset': response.prev_page.offset,
'backward_offset': response.next_page.offset}
logger.info("Initial sync point {}".format(self.initial_sync_point))
return response
elif not self.initial_sync_point:
raise ValueError
else:
assert 'descending' not in params
params['offset'] = self.initial_sync_point['forward_offset']
logger.info("Starting forward sync from offset {}".format(params['offset']))
return self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()})
def get_tenders(self, params={}, direction=""):
response = self.initialize_sync(params=params, direction=direction)
while not (params.get('descending') and not len(response.data) and params.get('offset') == response.next_page.offset):
tenders_list = response.data
params['offset'] = response.next_page.offset
delay = 101
if tenders_list:
delay = 15
logger.info("Client {} params: {}".format(direction, params))
for tender in tenders_list:
if tender['status'] in ("active.qualification",
"active.awarded", "complete"):
if hasattr(tender, "lots"):
if any([1 for lot in tender['lots'] if lot['status'] == "complete"]):
logger.info('{} sync: Found multilot tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_MULTILOT_COMPLETE}, {"TENDER_ID": tender['id']}))
yield tender
elif tender['status'] == "complete":
logger.info('{} sync: Found tender in complete status {}'.format(direction.capitalize(), tender['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_FOUND_NOLOT_COMPLETE}, {"TENDER_ID": tender['id']}))
yield tender
else:
logger.debug('{} sync: Skipping tender {} in status {}'.format(direction.capitalize(), tender['id'], tender['status']),
extra=journal_context(params={"TENDER_ID": tender['id']}))
logger.info('Sleep {} sync...'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_SLEEP}))
gevent.sleep(delay)
logger.info('Restore {} sync'.format(direction), extra=journal_context({"MESSAGE_ID": DATABRIDGE_SYNC_RESUME}))
logger.debug('{} {}'.format(direction, params))
response = self.tenders_sync_client.sync_tenders(params, extra_headers={'X-Client-Request-ID': generate_req_id()})
def get_tender_contracts(self):
while True:
try:
tender_to_sync = self.tenders_queue.get()
tender = self.tenders_sync_client.get_tender(tender_to_sync['id'],
extra_headers={'X-Client-Request-ID': generate_req_id()})['data']
db.put(tender_to_sync['id'], {'dateModified': tender_to_sync['dateModified']})
except Exception, e:
logger.exception(e)
logger.info('Put tender {} back to tenders queue'.format(tender_to_sync['id']), extra=journal_context(params={"TENDER_ID": tender_to_sync['id']}))
self.tenders_queue.put(tender_to_sync)
else:
if 'contracts' not in tender:
logger.warn('!!!No contracts found in tender {}'.format(tender['id']), extra=journal_context(params={"TENDER_ID": tender['id']}))
continue
for contract in tender['contracts']:
if contract["status"] == "active":
try:
if not db.has(contract['id']):
self.contracting_client.get_contract(contract['id'])
else:
logger.info('Contract {} exists in local db'.format(contract['id']), extra=journal_context(params={"CONTRACT_ID": contract['id']}))
continue
except ResourceNotFound:
logger.info('Sync contract {} of tender {}'.format(contract['id'], tender['id']), extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_CONTRACT_TO_SYNC}, {"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']}))
except Exception, e:
logger.exception(e)
logger.info('Put tender {} back to tenders queue'.format(tender_to_sync['id']), extra=journal_context(params={"TENDER_ID": tender_to_sync['id']}))
self.tenders_queue.put(tender_to_sync)
break
else:
db.put(contract['id'], True)
logger.info('Contract exists {}'.format(contract['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_CONTRACT_EXISTS},
{"CONTRACT_ID": contract['id']}))
continue
contract['tender_id'] = tender['id']
contract['procuringEntity'] = tender['procuringEntity']
if not contract.get('items'):
logger.info('Copying contract {} items'.format(contract['id']), extra=journal_context({"MESSAGE_ID": DATABRIDGE_COPY_CONTRACT_ITEMS},
{"CONTRACT_ID": contract['id']}))
if tender.get('lots'):
related_awards = [aw for aw in tender['awards'] if aw['id'] == contract['awardID']]
if related_awards:
award = related_awards[0]
if award.get("items"):
logger.debug('Copying items from related award {}'.format(award['id']))
contract['items'] = award['items']
else:
logger.debug('Copying items matching related lot {}'.format(award['relatedLot']))
contract['items'] = [item for item in tender['items'] if item['relatedLot'] == award['lotID']]
else:
logger.warn('Not found related award for contact {} of tender {}'.format(contract['id'], tender['id']),
extra=journal_context(params={"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']}))
else:
logger.debug('Copying all tender {} items into contract {}'.format(tender['id'], contract['id']),
extra=journal_context(params={"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']}))
contract['items'] = tender['items']
if not contract.get('items'):
logger.warn('Contact {} of tender {} does not contain items info'.format(contract['id'], tender['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_MISSING_CONTRACT_ITEMS},
{"CONTRACT_ID": contract['id'], "TENDER_ID": tender['id']}))
self.handicap_contracts_queue.put(contract)
class ContractingDataBridge(object):
""" Contracting Data Bridge """
def __init__(self, config):
super(ContractingDataBridge, self).__init__()
self.config = config
self.contracting_client = ContractingClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version')
)
params = {'opt_fields': 'status,lots',
'mode': '_all_'}
self.tenders_client_forward = TendersClient(
'',
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
params=params
)
params_desc = deepcopy(params)
params_desc['descending'] = 1
self.tenders_client_backward = TendersClient(
'',
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
params=params_desc
)
self.client = TendersClient(
self.config_get('api_token'),
host_url=self.config_get('tenders_api_server'),
api_version=self.config_get('tenders_api_version'),
)
self.tenders_queue = Queue()
self.handicap_contracts_queue = Queue()
self.contracts_put_queue = Queue()
self.contracts_retry_put_queue = Queue()
def config_get(self, name):
return self.config.get('main').get(name)
@retry(stop_max_attempt_number=5, wait_exponential_multiplier=1000)
def get_tender_credentials(self, tender_id):
return self.client.extract_credentials(tender_id)
def get_tenders(self, client):
while True:
request_id = generate_req_id()
client.headers.update({'X-Client-Request-ID': request_id})
tenders_list = list(client.get_tenders())
delay = 101
if tenders_list:
delay = 15
logger.info("Client params: {}".format(client.params))
for tender in tenders_list:
if tender['status'] in ("active.qualification",
"active.awarded", "complete"):
if hasattr(tender, "lots"):
if any([1 for lot in tender['lots'] if lot['status'] == "complete"]):
logger.info('found multilot tender {} in status {}'.format(tender['id'], tender['status']))
yield tender
elif tender['status'] == "complete":
logger.info('Found tender in complete status {}'.format(tender['id']))
yield tender
else:
logger.debug('Skipping tender {} in status {}'.format(tender['id'], tender['status']))
logger.info('Sleep...')
time.sleep(delay)
def get_tender_contracts(self):
while True:
request_id = generate_req_id()
self.tenders_client_backward.headers.update({'X-Client-Request-ID': request_id})
tender = self.tenders_client_backward.get_tender(self.tenders_queue.get()['id'])['data']
if 'contracts' not in tender:
logger.warn('!!!No contracts found in tender {}'.format(tender['id']))
continue
for contract in tender['contracts']:
if contract["status"] == "active":
try:
self.contracting_client.get_contract(contract['id'])
except ResourceNotFound:
logger.info('Sync contract {} of tender {}'.format(contract['id'], tender['id']))
else:
logger.info('Contract exists {}'.format(contract['id']))
continue
contract['tender_id'] = tender['id']
contract['procuringEntity'] = tender['procuringEntity']
self.handicap_contracts_queue.put(contract)
def prepare_contract_data(self):
while True:
contract = self.handicap_contracts_queue.get()
try:
logger.info("Getting extra info for tender {}".format(contract['tender_id']))
tender_data = self.get_tender_credentials(contract['tender_id'])
except Exception, e:
logger.info("Can't get tender credentials {}".format(contract['tender_id']))
self.handicap_contracts_queue.put(contract)
else:
logger.debug("Got extra info for tender {}".format(
contract['tender_id']))
data = tender_data.data
if data.get('mode'):
contract['mode'] = data['mode']
contract['owner'] = data['owner']
contract['tender_token'] = data['tender_token']
del contract['status'] # create contract in 'draft' status
self.contracts_put_queue.put(contract)
gevent.sleep(0)
