def setUp(self):
self.tender_id = uuid.uuid4().hex
self.award_id = uuid.uuid4().hex
self.qualification_id = uuid.uuid4().hex
self.document_id = generate_doc_id()
self.process_tracker = ProcessTracker(db=MagicMock())
self.process_tracker.set_item(self.tender_id, self.award_id, 1)
self.upload_to_tender_queue = Queue(10)
self.url = 'http://127.0.0.1:20604'
self.sleep_change_value = APIRateController()
self.data = Data(self.tender_id, self.award_id, '123', 'awards', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.qualification_data = Data(self.tender_id, self.qualification_id,
'123', 'qualifications', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.client = MagicMock()
self.worker = UploadFileToTender(self.client,
self.upload_to_tender_queue,
self.process_tracker, MagicMock(),
self.sleep_change_value)
self.worker.retry_upload_to_tender_queue = Queue(10)
def setUp(self):
self.tender_id = uuid.uuid4().hex
self.award_id = uuid.uuid4().hex
self.qualification_id = uuid.uuid4().hex
self.document_id = generate_doc_id()
self.process_tracker = ProcessTracker(db=MagicMock())
self.process_tracker.set_item(self.tender_id, self.award_id, 1)
self.upload_to_doc_service_queue = Queue(10)
self.upload_to_tender_queue = Queue(10)
self.sleep_change_value = APIRateController()
self.sna = event.Event()
self.sna.set()
self.data = Data(self.tender_id, self.award_id, '123', 'awards', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.qualification_data = Data(self.tender_id, self.qualification_id,
'123', 'qualifications', {
'meta': {
'id': self.document_id
},
'test_data': 'test_data'
})
self.doc_service_client = DocServiceClient(host='127.0.0.1',
port='80',
user='',
password='')
self.worker = UploadFileToDocService(self.upload_to_doc_service_queue,
self.upload_to_tender_queue,
self.process_tracker,
self.doc_service_client, self.sna,
self.sleep_change_value)
self.url = '{url}'.format(url=self.doc_service_client.url)
def temp_important_part_for_item(self, response, tender, item, item_name,
code):
self.process_tracker.set_item(tender['id'], item['id'])
document_id = generate_doc_id()
tender_data = Data(
tender['id'], item['id'], str(code), item_name + "s", {
'meta': {
'id': document_id,
'author': author,
'sourceRequests': [response.headers['X-Request-ID']]
}
})
self.edrpou_codes_queue.put(tender_data)
def prepare_data(self):
"""Get tender_id from filtered_tender_ids_queue, check award/qualification status, documentType; get
identifier's id and put into edrpou_codes_queue."""
while not self.exit:
self.services_not_available.wait()
try:
tender_id = self.filtered_tender_ids_queue.peek()
except LoopExit:
gevent.sleep(0)
continue
try:
response = self.tenders_sync_client.request(
"GET",
path='{}/{}'.format(self.tenders_sync_client.prefix_path,
tender_id),
headers={'X-Client-Request-ID': generate_req_id()})
except ResourceError as re:
if re.status_int == 429:
self.sleep_change_value.increment()
logger.info(
"Waiting tender {} for sleep_change_value: {} seconds".
format(tender_id,
self.sleep_change_value.time_between_requests))
else:
logger.warning(
'Fail to get tender info {}'.format(tender_id),
extra=journal_context(params={"TENDER_ID": tender_id}))
logger.exception("Message: {}, status_code {}".format(
re.msg, re.status_int))
logger.info(
'Leave tender {} in tenders queue'.format(tender_id),
extra=journal_context(params={"TENDER_ID": tender_id}))
gevent.sleep(0)
except Exception as e:
logger.warning(
'Fail to get tender info {}'.format(tender_id),
extra=journal_context(params={"TENDER_ID": tender_id}))
logger.exception("Message: {}".format(e.message))
logger.info(
'Leave tender {} in tenders queue'.format(tender_id),
extra=journal_context(params={"TENDER_ID": tender_id}))
gevent.sleep(0)
else:
self.sleep_change_value.decrement()
if response.status_int == 200:
tender = munchify(loads(response.body_string()))['data']
logger.info(
'Get tender {} from filtered_tender_ids_queue'.format(
tender_id),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_GET_TENDER_FROM_QUEUE},
params={"TENDER_ID": tender['id']}))
if 'awards' in tender:
for award in tender['awards']:
logger.info(
'Processing tender {} bid {} award {}'.format(
tender['id'], award['bid_id'], award['id']),
extra=journal_context(
{"MESSAGE_ID": DATABRIDGE_TENDER_PROCESS},
params={
"TENDER_ID": tender['id'],
"BID_ID": award['bid_id'],
"AWARD_ID": award['id']
}))
if self.should_process_item(award):
for supplier in award['suppliers']:
code = supplier['identifier']['id']
if self.is_code_invalid(code):
self.filtered_tender_ids_queue.get()
logger.info(
u'Tender {} bid {} award {} identifier id {} is not valid.'
.format(tender['id'], award['bid_id'],
award['id'], code),
extra=journal_context(
{
"MESSAGE_ID":
DATABRIDGE_TENDER_NOT_PROCESS
},
params={
"TENDER_ID": tender['id'],
"BID_ID": award['bid_id'],
"AWARD_ID": award['id']
}))
continue
# quick check if item was already processed
if self.process_tracker.check_processed_item(
tender['id'], award['id']):
logger.info(
'Tender {} bid {} award {} was already processed.'
.format(tender['id'], award['bid_id'],
award['id']),
extra=journal_context(
{
"MESSAGE_ID":
DATABRIDGE_TENDER_NOT_PROCESS
},
params={
"TENDER_ID": tender['id'],
"BID_ID": award['bid_id'],
"AWARD_ID": award['id']
}))
elif self.should_process_award(
supplier, tender, award):
self.process_tracker.set_item(
tender['id'], award['id'])
document_id = generate_doc_id()
tender_data = Data(
tender['id'], award['id'], str(code),
'awards', {
'meta': {
'id':
document_id,
'author':
author,
'sourceRequests': [
response.
headers['X-Request-ID']
]
}
})
self.edrpou_codes_queue.put(tender_data)
else:
logger.info(
'Tender {} bid {} award {} identifier schema isn\'t UA-EDR or tender is already in process.'
.format(tender['id'], award['bid_id'],
award['id']),
extra=journal_context(
{
"MESSAGE_ID":
DATABRIDGE_TENDER_NOT_PROCESS
},
params={
"TENDER_ID": tender['id'],
"BID_ID": award['bid_id'],
"AWARD_ID": award['id']
}))
else:
logger.info(
'Tender {} bid {} award {} is not in status pending or award has already document '
'with documentType registerExtract.'.format(
tender_id, award['bid_id'], award['id']),
extra=journal_context(
params={
"TENDER_ID": tender['id'],
"BID_ID": award['bid_id'],
"AWARD_ID": award['id']
}))
elif 'qualifications' in tender:
for qualification in tender['qualifications']:
if self.should_process_item(qualification):
appropriate_bid = [
b for b in tender['bids']
if b['id'] == qualification['bidID']
][0]
code = appropriate_bid['tenderers'][0][
'identifier']['id']
if self.is_code_invalid(code):
self.filtered_tender_ids_queue.get()
logger.info(
u'Tender {} bid {} qualification {} identifier id {} is not valid.'
.format(tender['id'],
qualification['bidID'],
qualification['id'], code),
extra=journal_context(
{
"MESSAGE_ID":
DATABRIDGE_TENDER_NOT_PROCESS
},
params={
"TENDER_ID": tender['id'],
"BID_ID": qualification['bidID'],
"QUALIFICATION_ID":
qualification['id']
}))
continue
# quick check if item was already processed
if self.process_tracker.check_processed_item(
tender['id'], qualification['id']):
logger.info(
'Tender {} bid {} qualification {} was already processed.'
.format(tender['id'],
qualification['bidID'],
qualification['id']),
extra=journal_context(
{
"MESSAGE_ID":
DATABRIDGE_TENDER_NOT_PROCESS
},
params={
"TENDER_ID": tender['id'],
"BID_ID": qualification['bidID'],
"QUALIFICATION_ID":
qualification['id']
}))
# check first identification scheme, if yes then check if item is already in process or not
elif self.should_process_qualification(
appropriate_bid, tender, qualification):
self.process_tracker.set_item(
tender['id'], qualification['id'])
document_id = generate_doc_id()
tender_data = Data(
tender['id'], qualification['id'],
str(code), 'qualifications', {
'meta': {
'id':
document_id,
'author':
author,
'sourceRequests':
[response.headers['X-Request-ID']]
}
})
self.edrpou_codes_queue.put(tender_data)
logger.info(
'Processing tender {} bid {} qualification {}'
.format(tender['id'],
qualification['bidID'],
qualification['id']),
extra=journal_context(
{
"MESSAGE_ID":
DATABRIDGE_TENDER_PROCESS
},
params={
"TENDER_ID": tender['id'],
"BID_ID": qualification['bidID'],
"QUALIFICATION_ID":
qualification['id']
}))
else:
logger.info(
'Tender {} bid {} qualification {} identifier schema is not UA-EDR or tender is already in process.'
.format(tender['id'],
qualification['bidID'],
qualification['id']),
extra=journal_context(
{
"MESSAGE_ID":
DATABRIDGE_TENDER_NOT_PROCESS
},
params={
"TENDER_ID": tender['id'],
"BID_ID": qualification['bidID'],
"QUALIFICATION_ID":
qualification['id']
}))
else:
logger.info(
'Tender {} bid {} qualification {} is not in status pending or qualification has '
'already document with documentType registerExtract.'
.format(tender_id, qualification['bidID'],
qualification['id']),
extra=journal_context(
params={
"TENDER_ID": tender['id'],
"BID_ID": qualification['bidID'],
"QUALIFICATION_ID": qualification['id']
}))
self.filtered_tender_ids_queue.get() # Remove elem from queue
gevent.sleep(self.sleep_change_value.time_between_requests)