def _run(self):
while not self.exit:
# Try get api client from clients queue
api_client_dict = self._get_api_client_dict()
if api_client_dict is None:
logger.critical('API clients queue is empty.')
sleep(self.config['worker_sleep'])
continue
# Try get item from resource items queue
priority, resource_item = self._get_resource_item_from_queue()
if resource_item is None:
self.api_clients_queue.put(api_client_dict)
logger.debug('PUT API CLIENT: {}'.format(
api_client_dict['id']),
extra={'MESSAGE_ID': 'put_client'})
logger.debug('Resource items queue is empty.')
sleep(self.config['worker_sleep'])
continue
handler = handlers_registry.get(
resource_item['procurementMethodType'], '')
if not handler:
handler = handlers_registry.get('common', '')
if not handler:
self.api_clients_queue.put(api_client_dict)
logger.critical(
"Not found handler for procurementMethodType: {}, {} {}".
format(resource_item['procurementMethodType'],
self.resource[:-1], resource_item['id']),
extra=journal_context(
{"MESSAGE_ID": "bridge_worker_exception"},
{self.input_resource_id: resource_item['id']}))
continue
# Try get resource item from public server
public_resource_item = self._get_resource_item_from_public(
api_client_dict, priority, resource_item)
if public_resource_item is None:
continue
try:
handler.process_resource(public_resource_item)
except RequestFailed as e:
logger.error(
"Error while processing {} {}: {}".format(
self.resource[:-1], resource_item['id'], e.message),
extra=journal_context(
{"MESSAGE_ID": "bridge_worker_exception"},
{self.input_resource_id: resource_item['id']}))
self.add_to_retry_queue(resource_item, priority)
except Exception as e:
logger.error(
"Error while processing {} {}: {}".format(
self.resource[:-1], resource_item['id'], e.message),
extra=journal_context(
{"MESSAGE_ID": "bridge_worker_exception"},
{self.input_resource_id: resource_item['id']}))
self.add_to_retry_queue(resource_item, priority)
def get_resource_credentials(self, resource_id):
self.input_client.headers.update(
{'X-Client-Request-ID': generate_req_id()})
logger.info("Getting credentials for tender {}".format(resource_id),
extra=journal_context(
{"MESSAGE_ID": "databridge_get_credentials"},
{"TENDER_ID": resource_id}))
data = self.input_client.extract_credentials(resource_id)
logger.info("Got tender {} credentials".format(resource_id),
extra=journal_context(
{"MESSAGE_ID": "databridge_got_credentials"},
{"TENDER_ID": resource_id}))
return data
def _run(self):
while INFINITY:
if not self.input_queue.empty():
priority, resource = self.input_queue.get()
else:
try:
priority, resource = self.input_queue.get(
timeout=self.timeout)
except Empty:
sleep(self.timeout)
continue
cached = self.cache_db.get(resource['id'])
if cached and cached == resource['dateModified']:
logger.info(
"{} {} not modified from last check. Skipping".format(
self.resource[:-1].title(), resource['id']),
extra=journal_context(
{"MESSAGE_ID": "SKIPPED"},
params={self.resource_id: resource['id']}))
continue
for re in self.filters:
if re.search(resource):
continue
else:
break
else:
logger.debug("Put to filtered queue {} {} {}".format(
self.resource[:-1], resource['id'], resource['status']))
self.filtered_queue.put((priority, resource))
continue
logger.info("Skip {} {}".format(self.resource[:-1],
resource['id']),
extra=journal_context(
{"MESSAGE_ID": "SKIPPED"},
params={self.resource_id: resource['id']}))
def _run(self):
while INFINITY:
if not self.input_queue.empty():
priority, resource = self.input_queue.get()
else:
try:
priority, resource = self.input_queue.get(
timeout=self.timeout)
except Empty:
sleep(self.timeout)
continue
cached = self.cache_db.get(resource['id'])
if cached and cached == resource['dateModified']:
LOGGER.info(
"{} {} not modified from last check. Skipping".format(
self.resource[:-1].title(), resource['id']),
extra=journal_context(
{"MESSAGE_ID": "SKIPPED"},
params={self.resource_id: resource['id']}))
continue
status = resource['status']
procurement_type = resource['procurementMethodType']
statuses_to_process = self.get_pmt_statuses(procurement_type)
if status not in statuses_to_process:
LOGGER.info("Skipping {} {} {} {}".format(
resource['procurementMethodType'], self.resource[:-1],
resource['status'], resource['id']),
extra=journal_context(
{"MESSAGE_ID": "SKIPPED"},
params={self.resource_id: resource['id']}))
continue
LOGGER.debug("Put to filtered queue {} {}".format(
self.resource[:-1], resource['id']))
self.filtered_queue.put((priority, resource))
def add_to_retry_queue(self, resource_item, priority=0, status_code=0):
retries_count = priority - 1000 if priority >= 1000 else priority
if retries_count > self.config['retries_count'] and status_code != 429:
logger.critical(
'{} {} reached limit retries count {} and droped from retry_queue.'
.format(self.resource[:-1].title(), resource_item['id'],
self.config['retries_count']),
extra=journal_context({'MESSAGE_ID': 'dropped_documents'},
{"TENDER_ID": resource_item['id']}))
return
timeout = 0
if status_code != 429:
timeout = self.config['retry_default_timeout'] * retries_count
priority += 1
sleep(timeout)
self.retry_resource_items_queue.put((priority, resource_item))
logger.info('Put to \'retry_queue\' {}: {}'.format(
self.resource[:-1], resource_item['id']),
extra={'MESSAGE_ID': 'add_to_retry'})
def test_journal_context(self):
self.assertEquals(journal_context(record={}, params={'test': 'test'}),
{'JOURNAL_test': 'test'})