def handle_response(
cls,
response: requests.models.Response) -> Generator[str, None, None]:
"""Handle a worker response.
Read all lines
Check if last line is OK
Then get the response file
And stream its contents
Finally delete the response file from the server
:param response: The response from the request made.
"""
worker_id = response.headers.get(cls._WORKER_ID_RESPONSE)
current_request_id = response.headers.get(cls._REQUEST_ID)
logger.info(
f"Worker response {worker_id} (request {current_request_id}) started."
)
last_line = None
for line in response.iter_lines():
last_line = line
last_line = last_line.decode()
if last_line == cls._WORKER_RESULT_FAILURE:
logger.info(
f"Worker response {worker_id} (request {current_request_id}) failed"
)
raise requests.exceptions.RequestException(
"Worker response failed")
elif last_line != cls._WORKER_RESULT_OK:
logger.info(
f"Worker response {worker_id} (request {current_request_id}) ended prematurely"
)
raise requests.exceptions.RequestException(
"Worker response ended prematurely")
else:
logger.info(
f"Worker result {worker_id} (request {current_request_id}) OK")
try:
# Request worker result
url = f"{cls._WORKER_API}/{worker_id}"
response = requests.get(url=url, stream=True)
response.raise_for_status()
for line in response.iter_lines():
yield line
except Exception as e:
logger.error(f"Worker result {worker_id} failed",
exc_info=True)
raise e
finally:
# Always try to cleanup worker files (even if an exception has occurred)
logger.info(
f"Worker result {worker_id} (request {current_request_id}) clear..."
)
url = f"{cls._WORKER_API}/end/{worker_id}"
response = requests.delete(url=url)
response.raise_for_status()
def raise_for_status(r: requests.models.Response) -> None:
try:
r.raise_for_status()
except requests.HTTPError as e:
try:
json = r.json()
except:
pass
else:
raise requests.HTTPError(json) from e
raise e
def isValidLink(link: requests.models.Response) -> bool:
"""
Returns true if a link is a valid webpage, false otherwise
"""
try:
link.raise_for_status()
except:
if DEBUGGING:
print("Error in %s." % (link))
return False
return True
def raise_for_status(r: requests.models.Response) -> None:
"""Attempts to include relevant JSON response info in exception raised by
"unsuccessful" request.
"""
try:
r.raise_for_status()
except requests.HTTPError as e:
try:
json = r.json()
except:
pass
else:
raise requests.HTTPError(json) from e
raise e
def _raise_for_status(self, response: requests.models.Response) -> None:
try:
response.raise_for_status()
except HTTPError:
self.log.error(response.json().get('exception'))
raise
def check_response_status_vk(response: requests.models.Response):
response.raise_for_status()
response_data = response.json()
if 'error' in response_data:
raise requests.exceptions.HTTPError(response_data['error'])
return response_data