def upload_to_doc_service(self, data, tender_id, item_name, item_id):
# check if the file has been already uploaded
# will retry the task until mongodb returns either doc or None
unique_data = {k: v for k, v in data.items() if k != "meta"}
upload_results = get_upload_results(self, unique_data, tender_id,
item_name, item_id)
if upload_results is None:
# generate file data
contents = yaml.safe_dump(data,
allow_unicode=True,
default_flow_style=False)
temporary_file = io.StringIO(contents)
temporary_file.name = FILE_NAME
files = {'file': (FILE_NAME, temporary_file, 'application/yaml')}
try:
response = requests.post(
'{host}/upload'.format(host=DS_HOST),
auth=(DS_USER, DS_PASSWORD),
timeout=(CONNECT_TIMEOUT, READ_TIMEOUT),
files=files,
headers={'X-Client-Request-ID': data['meta']['id']})
except RETRY_REQUESTS_EXCEPTIONS as exc:
logger.exception(exc,
extra={"MESSAGE_ID": "EDR_POST_DOC_EXCEPTION"})
raise self.retry(exc=exc)
else:
if response.status_code != 200:
logger_method = get_task_retry_logger_method(self, logger)
logger_method("Incorrect upload status for doc {}".format(
data['meta']['id']),
extra={
"MESSAGE_ID": "EDR_POST_DOC_ERROR",
"STATUS_CODE": response.status_code,
})
raise self.retry(
countdown=get_request_retry_countdown(response))
response_json = response.json()
response_json['meta'] = {'id': data['meta']['id']}
# save response to mongodb, so that the file won't be uploaded again
# fail silently: if mongodb isn't available, the task will neither fail nor retry
# in worst case there might be a duplicate attached to the tender
uid = save_upload_results(response_json, unique_data, tender_id,
item_name, item_id)
logger.info("Saved document with uid {} for {} {} {}".format(
uid, tender_id, item_name, item_id),
extra={"MESSAGE_ID": "EDR_POST_UPLOAD_RESULTS_SUCCESS"})
else:
# we don't need to pass the response since it's saved to mongodb doc
response_json = None
attach_doc_to_tender.delay(file_data=response_json,
data=data,
tender_id=tender_id,
item_name=item_name,
item_id=item_id)
def process_tender(self, tender_id, *args, **kwargs):
url = "{host}/api/{version}/tenders/{tender_id}".format(
host=PUBLIC_API_HOST,
version=API_VERSION,
tender_id=tender_id,
)
try:
response = requests.get(
url,
headers={"X-Client-Request-ID": uuid4().hex},
timeout=(CONNECT_TIMEOUT, READ_TIMEOUT),
)
except RETRY_REQUESTS_EXCEPTIONS as exc:
logger.exception(exc, extra={"MESSAGE_ID": "EDR_GET_TENDER_EXCEPTION"})
raise self.retry(exc=exc)
else:
if response.status_code != 200:
logger_method = get_task_retry_logger_method(self, logger)
logger_method(
"Unexpected status code {} while getting tender {}".format(
response.status_code, tender_id),
extra={
"MESSAGE_ID": "EDR_GET_TENDER_CODE_ERROR",
"STATUS_CODE": response.status_code,
})
raise self.retry(countdown=get_request_retry_countdown(response))
tender_data = response.json()["data"]
if not should_process_tender(tender_data):
return
# --------
i = 0 # spread in time tasks that belongs to a single tender CS-3854
if 'awards' in tender_data:
for award in tender_data['awards']:
if should_process_item(award):
for supplier in award['suppliers']:
process_award_supplier(response, tender_data, award,
supplier, i)
i += 1
elif 'qualifications' in tender_data:
for qualification in tender_data['qualifications']:
if should_process_item(qualification):
process_qualification(response, tender_data, qualification,
i)
i += 1
def process_feed(self,
resource="tenders",
offset=None,
descending=None,
mode="_all_",
cookies=None,
try_count=0):
logger.info("Start task {}".format(self.request.id),
extra={
"MESSAGE_ID": "START_TASK_MSG",
"TASK_ID": self.request.id
})
config = resources.configs.get(resource)
cookies = cookies or {}
if not offset: # initialization
descending = "1"
url = FEED_URL_TEMPLATE.format(
host=PUBLIC_API_HOST,
version=API_VERSION,
resource=resource,
)
params = dict(
feed="changes",
limit=API_LIMIT,
mode=mode,
)
if config.opt_fields:
params["opt_fields"] = ",".join(config.opt_fields)
if descending:
params["descending"] = descending
if offset:
params["offset"] = offset
try:
response = requests.get(
url,
params=params,
cookies=cookies,
timeout=(CONNECT_TIMEOUT, READ_TIMEOUT),
)
except RETRY_REQUESTS_EXCEPTIONS as exc:
logger.exception(exc, extra={"MESSAGE_ID": "FEED_RETRY_EXCEPTION"})
raise self.retry(exc=exc)
else:
if response.status_code == 200:
# handle cookies
if response.cookies:
cookies = response.cookies.get_dict()
# get response data
response_json = response.json()
# call handlers (TENDER_HANDLERS, CONTRACT_HANDLERS, FRAMEWORK_HANDLERS
item_handlers = config.handlers
for item in response_json["data"]:
for handler in item_handlers:
try:
handler(item)
except Exception as e:
logger.exception(
e, extra={"MESSAGE_ID": "FEED_HANDLER_EXCEPTION"})
# schedule getting the next page
next_page_kwargs = dict(
resource=resource,
mode=mode,
offset=response_json["next_page"]["offset"],
cookies=cookies)
if descending:
next_page_kwargs["descending"] = descending
if len(response_json["data"]) < API_LIMIT:
if descending:
logger.info("Stopping backward crawling",
extra={"MESSAGE_ID": "FEED_BACKWARD_FINISH"})
else:
if offset == next_page_kwargs["offset"]:
# increase try_count so task won't be stopped by unique_lock decorator
next_page_kwargs["try_count"] = try_count + 1
else:
# reset try_count to sync all duplicate tasks
# and let unique_lock decorator do it's job
next_page_kwargs.pop("try_count", None)
process_feed.apply_async(
kwargs=next_page_kwargs,
countdown=WAIT_MORE_RESULTS_COUNTDOWN,
)
else:
process_feed.apply_async(kwargs=next_page_kwargs)
# if it's initialization, add forward crawling task
if not offset:
process_kwargs = dict(
resource=resource,
mode=mode,
cookies=cookies,
)
if response_json.get("prev_page", {}).get("offset"):
process_kwargs["offset"] = response_json["prev_page"][
"offset"]
else:
logger.debug("Initialization on an empty feed result",
extra={"MESSAGE_ID": "FEED_INIT_EMPTY"})
process_kwargs["try_count"] = try_count + 1
process_feed.apply_async(
kwargs=process_kwargs,
countdown=WAIT_MORE_RESULTS_COUNTDOWN,
)
elif response.status_code == 412: # Precondition failed
logger.warning(
"Precondition failed with cookies {}".format(cookies),
extra={"MESSAGE_ID": "FEED_PRECONDITION_FAILED"})
retry_kwargs = dict(**self.request.kwargs)
retry_kwargs["cookies"] = response.cookies.get_dict()
raise self.retry(kwargs=retry_kwargs)
elif response.status_code == 404: # "Offset expired/invalid"
logger.warning("Offset {} failed with cookies {}".format(
offset, cookies),
extra={"MESSAGE_ID": "FEED_OFFSET_FAILED"})
if not descending or not offset: # for forward process only
logger.info("Feed process reinitialization",
extra={"MESSAGE_ID": "FEED_REINITIALIZATION"})
retry_kwargs = {
k: v
for k, v in self.request.kwargs.items() if k != "offset"
}
raise self.retry(kwargs=retry_kwargs)
else:
logger.warning("Unexpected status code {}: {}".format(
response.status_code, response.text),
extra={"MESSAGE_ID": "FEED_UNEXPECTED_STATUS"})
raise self.retry(countdown=get_request_retry_countdown(response))
return response.status_code
def attach_doc_to_tender(self, file_data, data, tender_id, item_name, item_id):
unique_data = {k: v for k, v in data.items() if k != "meta"}
upload_results = get_upload_results(self, unique_data, tender_id,
item_name, item_id)
if file_data is None:
if upload_results is None:
fall_msg = "Saved results are missed for {} {} {}".format(
tender_id, item_name, item_id)
logger.critical(fall_msg,
extra={"MESSAGE_ID": "EDR_SAVED_RESULTS_MISSED"})
raise AssertionError(fall_msg)
else:
file_data = upload_results["file_data"]
if upload_results and upload_results.get("attached"):
logger.info(
"Uploaded file has been already attached to the tender: {} {} {}".
format(tender_id, item_name, item_id),
extra={"MESSAGE_ID": "EDR_FILE_ALREADY_ATTACHED"})
return
document_data = file_data['data']
document_data["documentType"] = DOC_TYPE
url = "{host}/api/{version}/tenders/{tender_id}/{item_name}s/{item_id}/documents".format(
host=API_HOST,
version=API_VERSION,
item_name=item_name,
item_id=item_id,
tender_id=tender_id,
)
meta_id = file_data['meta']['id']
# get SERVER_ID cookie
try:
head_response = requests.head(url,
timeout=(CONNECT_TIMEOUT, READ_TIMEOUT),
headers={
'Authorization':
'Bearer {}'.format(API_TOKEN),
'X-Client-Request-ID':
meta_id,
})
except RETRY_REQUESTS_EXCEPTIONS as exc:
logger.exception(exc,
extra={"MESSAGE_ID": "EDR_ATTACH_DOC_HEAD_EXCEPTION"})
raise self.retry(exc=exc)
else:
# post document
try:
response = requests.post(
url,
json={'data': document_data},
timeout=(CONNECT_TIMEOUT, READ_TIMEOUT),
headers={
'Authorization': 'Bearer {}'.format(API_TOKEN),
'X-Client-Request-ID': meta_id,
},
cookies=head_response.cookies,
)
except RETRY_REQUESTS_EXCEPTIONS as exc:
logger.exception(
exc, extra={"MESSAGE_ID": "EDR_ATTACH_DOC_POST_EXCEPTION"})
raise self.retry(exc=exc)
else:
# handle response code
if response.status_code == 422:
logger.error(
"Incorrect document data while attaching doc {} to tender {}: {}"
.format(meta_id, tender_id, response.text),
extra={"MESSAGE_ID": "EDR_ATTACH_DATA_ERROR"})
elif response.status_code != 201:
logger.warning(
"Incorrect upload status while attaching doc {} to tender {}"
.format(meta_id, tender_id),
extra={
"MESSAGE_ID": "EDR_ATTACH_STATUS_ERROR",
"STATUS_CODE": response.status_code,
})
raise self.retry(
countdown=get_request_retry_countdown(response))
else:
# won't raise anything
uid = set_upload_results_attached(unique_data, tender_id,
item_name, item_id)
logger.info(
"Set attached document with uid {} for {} {} {}".format(
uid, tender_id, item_name, item_id),
extra={"MESSAGE_ID": "EDR_SET_ATTACHED_RESULTS"})
def test_wrong_value_type(self):
request = Mock()
countdown = get_request_retry_countdown(request)
self.assertEqual(countdown, DEFAULT_RETRY_AFTER)
def test_float(self):
request = Mock()
request.headers = {"Retry-After": 130.1}
countdown = get_request_retry_countdown(request)
self.assertEqual(countdown, 130.1)
def test_string(self):
request = Mock()
request.headers = {"Retry-After": "12.5"}
countdown = get_request_retry_countdown(request)
self.assertEqual(countdown, 12.5)
def test_wrong_value(self):
request = Mock()
request.headers = {"Retry-After": "sundown"}
countdown = get_request_retry_countdown(request)
self.assertEqual(countdown, DEFAULT_RETRY_AFTER)
def test_no_value(self):
request = Mock()
request.headers = {}
countdown = get_request_retry_countdown(request)
self.assertEqual(countdown, DEFAULT_RETRY_AFTER)