def get_online_logs(logs):
""" Try to get online logs and add it to attachments
:param `worker.parsing.parser.ParsedEmail` parsed_email: The parsed email
:rtype: dict
:returns: The attachment of None if failed to retreive
"""
attachment = {'content_type': 'text/plain', 'filename': 'online_logs.txt'}
try:
response = utils.request_wrapper(logs[0], method='GET', as_json=False)
soup = BeautifulSoup(response.text)
for script in soup(['script', 'style']):
script.extract()
text = soup.get_text()
text = re.sub('\n\n+', '\n\n', text)
text = re.sub(' +', ' ', text)
attachment['content'] = text.encode('utf-8')
return attachment
except (UnicodeDecodeError, UnicodeEncodeError, utils.RequestException):
return None
def test_request_wrapper(self, mock_urlopen):
mocked_response = Response()
mocked_response.status_code = 500
mock_urlopen.return_value = mocked_response
self.assertRaises(utils.RequestException, lambda: utils.request_wrapper('https://www.ovh.com', method='GET'))