def initialize_sync(self, params=None, direction=None):
if direction == "backward":
self.initialization_event.clear()
assert params['descending']
response = self.tenders_sync_client.sync_tenders(
params,
extra_headers={'X-Client-Request-ID': generate_req_id()})
# set values in reverse order due to 'descending' option
self.initial_sync_point = {
'forward_offset': response.prev_page.offset,
'backward_offset': response.next_page.offset
}
self.initialization_event.set() # wake up forward worker
logger.info("Initial sync point {}".format(
self.initial_sync_point))
return response
else:
assert 'descending' not in params
self.initialization_event.wait()
params['offset'] = self.initial_sync_point['forward_offset']
logger.info("Starting forward sync from offset {}".format(
params['offset']))
return self.tenders_sync_client.sync_tenders(
params,
extra_headers={'X-Client-Request-ID': generate_req_id()})
def get_tenders(self, params={}, direction=""):
response = self.initialize_sync(params=params, direction=direction)
while not (params.get('descending') and not len(response.data)
and params.get('offset') == response.next_page.offset):
tenders = response.data if response else []
params['offset'] = response.next_page.offset
for tender in tenders:
if self.should_process_tender(tender):
yield tender
else:
logger.info(
'Skipping tender {} with status {} with procurementMethodType {}'
.format(tender['id'], tender['status'],
tender['procurementMethodType']),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_INFO},
params={"TENDER_ID": tender['id']}))
logger.info('Sleep {} sync...'.format(direction),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_SYNC_SLEEP}))
gevent.sleep(self.delay +
self.sleep_change_value.time_between_requests)
try:
response = self.tenders_sync_client.sync_tenders(
params,
extra_headers={'X-Client-Request-ID': generate_req_id()})
self.sleep_change_value.decrement()
except ResourceError as re:
if re.status_int == 429:
self.sleep_change_value.increment()
logger.info("Received 429, will sleep for {}".format(
self.sleep_change_value.time_between_requests))
else:
raise re
def temp_process(self, tender_id):
try:
response = self.tenders_sync_client.request(
"GET",
path='{}/{}'.format(self.tenders_sync_client.prefix_path,
tender_id),
headers={'X-Client-Request-ID': generate_req_id()})
except Exception as e:
if getattr(e, "status_int", False) == 429:
self.sleep_change_value.increment()
logger.info(
"Waiting tender {} for sleep_change_value: {} seconds".
format(tender_id,
self.sleep_change_value.time_between_requests))
else:
logger.warning(
'Fail to get tender info {}'.format(tender_id),
extra=journal_context(params={"TENDER_ID": tender_id}))
logger.exception("Message: {}".format(e.message))
else:
self.process_items_and_move(response, tender_id)
def prepare_data(self):
"""Get tender_id from filtered_tender_ids_queue, check award/qualification status, documentType; get
identifier's id and put into edrpou_codes_queue."""
while not self.exit:
self.services_not_available.wait()
try:
tender_id = self.filtered_tender_ids_queue.peek()
except LoopExit:
gevent.sleep(0)
continue
try:
response = self.tenders_sync_client.request(
"GET",
path='{}/{}'.format(self.tenders_sync_client.prefix_path,
tender_id),
headers={'X-Client-Request-ID': generate_req_id()})
except ResourceError as re:
if re.status_int == 429:
self.sleep_change_value.increment()
logger.info(
"Waiting tender {} for sleep_change_value: {} seconds".
format(tender_id,
self.sleep_change_value.time_between_requests))
else:
logger.warning(
'Fail to get tender info {}'.format(tender_id),
extra=journal_context(params={"TENDER_ID": tender_id}))
logger.exception("Message: {}, status_code {}".format(
re.msg, re.status_int))
logger.info(
'Leave tender {} in tenders queue'.format(tender_id),
extra=journal_context(params={"TENDER_ID": tender_id}))
gevent.sleep(0)
except Exception as e:
logger.warning(
'Fail to get tender info {}'.format(tender_id),
extra=journal_context(params={"TENDER_ID": tender_id}))
logger.exception("Message: {}".format(e.message))
logger.info(
'Leave tender {} in tenders queue'.format(tender_id),
extra=journal_context(params={"TENDER_ID": tender_id}))
gevent.sleep(0)
else:
self.sleep_change_value.decrement()
if response.status_int == 200:
tender = munchify(loads(response.body_string()))['data']
logger.info(
'Get tender {} from filtered_tender_ids_queue'.format(
tender_id),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_GET_TENDER_FROM_QUEUE},
params={"TENDER_ID": tender['id']}))
if 'awards' in tender:
for award in tender['awards']:
logger.info(
'Processing tender {} bid {} award {}'.format(
tender['id'], award['bid_id'], award['id']),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_TENDER_PROCESS},
params={
"TENDER_ID": tender['id'],
"BID_ID": award['bid_id'],
"AWARD_ID": award['id']
}))
if self.should_process_item(award):
for supplier in award['suppliers']:
code = supplier['identifier']['id']
if self.is_code_invalid(code):
self.filtered_tender_ids_queue.get()
logger.info(
u'Tender {} bid {} award {} identifier id {} is not valid.'
.format(tender['id'], award['bid_id'],
award['id'], code),
extra=journal_context(
{
"MESSAGE_ID":
DATABRIDGE_TENDER_NOT_PROCESS
},
params={
"TENDER_ID": tender['id'],
"BID_ID": award['bid_id'],
"AWARD_ID": award['id']
}))
continue
# quick check if item was already processed
if self.process_tracker.check_processed_item(
tender['id'], award['id']):
logger.info(
'Tender {} bid {} award {} was already processed.'
.format(tender['id'], award['bid_id'],
award['id']),
extra=journal_context(
{
"MESSAGE_ID":
DATABRIDGE_TENDER_NOT_PROCESS
},
params={
"TENDER_ID": tender['id'],
"BID_ID": award['bid_id'],
"AWARD_ID": award['id']
}))
elif self.should_process_award(
supplier, tender, award):
self.process_tracker.set_item(
tender['id'], award['id'])
document_id = generate_doc_id()
tender_data = Data(
tender['id'], award['id'], str(code),
'awards', {
'meta': {
'id':
document_id,
'author':
author,
'sourceRequests': [
response.
headers['X-Request-ID']
]
}
})
self.edrpou_codes_queue.put(tender_data)
else:
logger.info(
'Tender {} bid {} award {} identifier schema isn\'t UA-EDR or tender is already in process.'
.format(tender['id'], award['bid_id'],
award['id']),
extra=journal_context(
{
"MESSAGE_ID":
DATABRIDGE_TENDER_NOT_PROCESS
},
params={
"TENDER_ID": tender['id'],
"BID_ID": award['bid_id'],
"AWARD_ID": award['id']
}))
else:
logger.info(
'Tender {} bid {} award {} is not in status pending or award has already document '
'with documentType registerExtract.'.format(
tender_id, award['bid_id'], award['id']),
extra=journal_context(
params={
"TENDER_ID": tender['id'],
"BID_ID": award['bid_id'],
"AWARD_ID": award['id']
}))
elif 'qualifications' in tender:
for qualification in tender['qualifications']:
if self.should_process_item(qualification):
appropriate_bid = [
b for b in tender['bids']
if b['id'] == qualification['bidID']
][0]
code = appropriate_bid['tenderers'][0][
'identifier']['id']
if self.is_code_invalid(code):
self.filtered_tender_ids_queue.get()
logger.info(
u'Tender {} bid {} qualification {} identifier id {} is not valid.'
.format(tender['id'],
qualification['bidID'],
qualification['id'], code),
extra=journal_context(
{
"MESSAGE_ID":
DATABRIDGE_TENDER_NOT_PROCESS
},
params={
"TENDER_ID": tender['id'],
"BID_ID": qualification['bidID'],
"QUALIFICATION_ID":
qualification['id']
}))
continue
# quick check if item was already processed
if self.process_tracker.check_processed_item(
tender['id'], qualification['id']):
logger.info(
'Tender {} bid {} qualification {} was already processed.'
.format(tender['id'],
qualification['bidID'],
qualification['id']),
extra=journal_context(
{
"MESSAGE_ID":
DATABRIDGE_TENDER_NOT_PROCESS
},
params={
"TENDER_ID": tender['id'],
"BID_ID": qualification['bidID'],
"QUALIFICATION_ID":
qualification['id']
}))
# check first identification scheme, if yes then check if item is already in process or not
elif self.should_process_qualification(
appropriate_bid, tender, qualification):
self.process_tracker.set_item(
tender['id'], qualification['id'])
document_id = generate_doc_id()
tender_data = Data(
tender['id'], qualification['id'],
str(code), 'qualifications', {
'meta': {
'id':
document_id,
'author':
author,
'sourceRequests':
[response.headers['X-Request-ID']]
}
})
self.edrpou_codes_queue.put(tender_data)
logger.info(
'Processing tender {} bid {} qualification {}'
.format(tender['id'],
qualification['bidID'],
qualification['id']),
extra=journal_context(
{
"MESSAGE_ID":
DATABRIDGE_TENDER_PROCESS
},
params={
"TENDER_ID": tender['id'],
"BID_ID": qualification['bidID'],
"QUALIFICATION_ID":
qualification['id']
}))
else:
logger.info(
'Tender {} bid {} qualification {} identifier schema is not UA-EDR or tender is already in process.'
.format(tender['id'],
qualification['bidID'],
qualification['id']),
extra=journal_context(
{
"MESSAGE_ID":
DATABRIDGE_TENDER_NOT_PROCESS
},
params={
"TENDER_ID": tender['id'],
"BID_ID": qualification['bidID'],
"QUALIFICATION_ID":
qualification['id']
}))
else:
logger.info(
'Tender {} bid {} qualification {} is not in status pending or qualification has '
'already document with documentType registerExtract.'
.format(tender_id, qualification['bidID'],
qualification['id']),
extra=journal_context(
params={
"TENDER_ID": tender['id'],
"BID_ID": qualification['bidID'],
"QUALIFICATION_ID": qualification['id']
}))
self.filtered_tender_ids_queue.get() # Remove elem from queue
gevent.sleep(self.sleep_change_value.time_between_requests)