def get_tenders(self, params={}, direction=""):
response = self.initialize_sync(params=params, direction=direction)
while more_tenders(params, response):
tenders = response.data if response else []
params['offset'] = response.next_page.offset
for tender in tenders:
if self.should_process_tender(tender):
yield tender
else:
logger.debug(
'Skipping tender {} with status {} with procurementMethodType {}'
.format(tender['id'], tender['status'],
tender['procurementMethodType']),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_INFO},
params={"TENDER_ID": tender['id']}))
logger.debug('Sleep {} sync...'.format(direction),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_SYNC_SLEEP}))
gevent.sleep(self.delay +
self.sleep_change_value.time_between_requests)
try:
response = self.tenders_sync_client.sync_tenders(
params,
extra_headers={'X-Client-Request-ID': generate_req_id()})
self.sleep_change_value.decrement()
except ResourceError as re:
if re.status_int == 429:
self.sleep_change_value.increment()
logger.info("Received 429, will sleep for {}".format(
self.sleep_change_value.time_between_requests))
else:
raise re
def prepare_data(self):
"""Get tender_id from filtered_tender_ids_queue, check award/qualification status, documentType; get
identifier's id and put into edrpou_codes_queue."""
while not self.exit:
self.services_not_available.wait()
try:
tender_id = self.filtered_tender_ids_queue.peek()
except LoopExit:
gevent.sleep()
continue
try:
response = self.tenders_sync_client.request("GET",
path='{}/{}'.format(self.tenders_sync_client.prefix_path,
tender_id),
headers={'X-Client-Request-ID': generate_req_id()})
except Exception as e:
if getattr(e, "status_int", False) == 429:
self.sleep_change_value.increment()
logger.info("Waiting tender {} for sleep_change_value: {} seconds".format(
tender_id, self.sleep_change_value.time_between_requests))
else:
logger.warning('Fail to get tender info {}. Message {}'.format(tender_id, e.message),
extra=journal_context(params={"TENDER_ID": tender_id}))
gevent.sleep()
else:
self.sleep_change_value.decrement()
if response.status_int == 200:
self.process_response(response)
else:
logger.warning('Fail to get tender info {}'.format(tender_id),
extra=journal_context(params={"TENDER_ID": tender_id}))
self.filtered_tender_ids_queue.get()
gevent.sleep(self.sleep_change_value.time_between_requests)
def move_to_tender_queue(self, tender, response):
data = tender
data.file_content = dict(response.json(), **{'meta': {'id': tender.doc_id()}})
self.upload_to_tender_queue.put(data)
logger.info(u'Successfully uploaded file to doc service {} doc_id: {}'.format(tender, tender.doc_id()),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_UPLOAD_TO_DOC_SERVICE},
tender.log_params()))
def put_tenders_to_process(self, params, direction):
for tender in self.get_tenders(params=params, direction=direction):
logger.debug('Backward sync: Put tender {} to process...'.format(
tender['id']),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_TENDER_PROCESS},
{"TENDER_ID": tender['id']}))
self.filtered_tender_ids_queue.put(tender['id'])
def move_data_to_retry_or_leave(self, response, doc_and_tenders, is_retry):
logger.info('Unsuccessful response from document service while uploading {} doc_id: {}. Response {}'.
format(doc_and_tenders, doc_and_tenders[0]["meta"]["id"], response.status_code),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE},
doc_and_tenders[1][0].log_params()))
if not is_retry:
self.retry_upload_to_doc_service_queue.put(doc_and_tenders)
self.upload_to_doc_service_queue.get()
def handle_error(self, re, tender_data, is_retry):
logger.info('Error while uploading file to {} doc_id: {}. Status: {}. Message: {}'.format(
tender_data, tender_data.doc_id(), getattr(re, "status_int", None), re.message),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_TENDER}, tender_data.log_params()))
self.sleep_change_value.decrement()
if not is_retry:
self.retry_upload_to_tender_queue.put(tender_data)
self.upload_to_tender_queue.get()
def revive_job(self, name):
logger.warning(
"{} dead try restart".format(name),
extra=journal_context(
{"MESSAGE_ID": 'DATABRIDGE_RESTART_{}'.format(name.lower())},
{}))
self.immortal_jobs[name] = gevent.spawn(getattr(self, name))
logger.info("{} is up".format(name))
def process_response(self, response):
tender = loads(response.body_string())['data']
for aw in active_award(tender):
logger.info("active award {}".format(active_award(tender)))
for code in get_codes(aw):
logger.info("code {}".format(code))
data = Data(tender['id'], aw['id'], code[0], code[1],
file_content={"meta": {'sourceRequests': [response.headers['X-Request-ID']]}})
self.process_tracker.set_item(data.tender_id, data.award_id)
self.edrpou_codes_queue.put(data)
logger.info(u"Have put {} into edrpou_codes_queue".format(data))
else:
logger.info('Tender {} bid {} award {} identifier schema isn\'t UA-EDR.'.format(
tender['id'], aw['bid_id'], aw['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_NOT_PROCESS},
journal_item_params(tender['id'], aw['bid_id'], aw['id'])))
break
else:
logger.info('Tender {} is already in process or was processed.'.format(tender['id']),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_TENDER_NOT_PROCESS},
{"TENDER_ID": tender['id']}))
def _run(self):
self.services_not_available.wait()
logger.info('Start {} worker'.format(type(self).__name__),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_START_UPLOAD}, {}))
self.immortal_jobs = self._start_jobs()
try:
while not self.exit:
gevent.sleep(self.delay)
self.check_and_revive_jobs()
except Exception as e:
logger.error(e)
gevent.killall(self.immortal_jobs.values(), timeout=5)
def remove_bad_data(self, doc_and_tenders, e, is_retry):
logger.exception('Exception while uploading file to doc service {} doc_id: {}. Message: {}. {}'.
format(doc_and_tenders, doc_and_tenders[0]["meta"]["id"], e, "Removed tender data" if is_retry else ""),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_UNSUCCESS_UPLOAD_TO_DOC_SERVICE},
doc_and_tenders[1][0].log_params()))
if is_retry:
self.retry_upload_to_doc_service_queue.get()
self.process_tracker.update_items_and_tender(doc_and_tenders[1][0].tender_id,
doc_and_tenders[1][0].award_id,
doc_and_tenders[1][0].doc_id())
raise e
else:
self.retry_upload_to_doc_service_queue.put(doc_and_tenders)
self.upload_to_doc_service_queue.get()
def get_tenders_forward(self):
self.services_not_available.wait()
logger.info('Start forward data sync worker...')
params = {
'opt_fields': 'status,procurementMethodType',
'mode': '_all_'
}
try:
self.put_tenders_to_process(params, "forward")
except Exception as e:
logger.warning('Forward worker died!',
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_WORKER_DIED}, {}))
logger.exception("Message: {}".format(e.message))
else:
logger.warning('Forward data sync finished!')
def successfully_uploaded_to_tender(self, tender_data, is_retry):
logger.info('Successfully uploaded file to {} doc_id: {}'.format(tender_data, tender_data.doc_id()),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_SUCCESS_UPLOAD_TO_TENDER},
tender_data.log_params()))
self.remove_data(tender_data, is_retry)
def decrease_request_frequency(self, re, tender_data):
logger.info("Accept 429 while uploading to {} doc_id: {}. Message {}".format(
tender_data, tender_data.doc_id(), re.msg),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING},
tender_data.log_params()))
self.sleep_change_value.increment()
def removing_data(self, re, tender_data, is_retry):
logger.warning("Accept {} while uploading to {} doc_id: {}. Message {}".format(
re.status_int, tender_data, tender_data.doc_id(), re.msg),
extra=journal_context({"MESSAGE_ID": DATABRIDGE_ITEM_STATUS_CHANGED_WHILE_PROCESSING},
tender_data.log_params()))
self.remove_data(tender_data, is_retry)