def test_dispatch_event__handle_request_exception(self):
""" Test that dispatch event handles exceptions and logs error. """
url = 'https://www.optimizely.com'
params = {
'accountId': '111001',
'eventName': 'test_event',
'eventEntityId': '111028',
'visitorId': 'oeutest_user',
}
event = event_builder.Event(
url,
params,
http_verb='POST',
headers={'Content-Type': 'application/json'})
with mock.patch(
'requests.post',
side_effect=request_exception.RequestException(
'Failed Request'),
) as mock_request_post, mock.patch('logging.error') as mock_log_error:
event_dispatcher.EventDispatcher.dispatch_event(event)
mock_request_post.assert_called_once_with(
url,
data=json.dumps(params),
headers={'Content-Type': 'application/json'},
timeout=event_dispatcher.REQUEST_TIMEOUT,
)
mock_log_error.assert_called_once_with(
'Dispatch event failed. Error: Failed Request')
def test_all_request_servers_error(self, mock_request):
mock_request.side_effect = exceptions.RequestException(
'mock exception')
# will try each server, and mark each down on a 500 response
# when no servers are left, a NoReachableServersException is thrown
self.assertRaises(NoReachableServers, self._service.request, 'GET',
'/relative/path')
# should have tried twice - once for each server
self.assertEqual(2, mock_request.call_count)
# add our hostnames to a set (to ensure the
# same server isn't being used repeatedly)
# all other info should be the same for each server request
hostnames = set()
for i, call_arg in enumerate(mock_request.call_args_list):
url = UrlBuilder.from_url(call_arg[0][1])
hostnames.add(url._hostname)
self.assertEqual(url._scheme, 'https')
self.assertIsNone(url._port)
self.assertEqual(url._path, '/relative/path')
# less than 2 means a server has been reused
self.assertEqual(2, len(hostnames))
for hostname in hostnames:
self.assertIn(hostname, _EXPECTED_SERVERS)
def test_swarm_validate_discovery_url_fail(self, mock_get):
mock_get.side_effect = req_exceptions.RequestException()
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
self.assertRaises(exception.GetClusterSizeFailed,
k8s_def.validate_discovery_url, 'http://etcd/test',
1)
def __handle_response(self, action, url, raw=False, **kwargs):
"""Handle the response after making a HTTP request and throw any exceptions if the status is not 20x.
:param action: The Session method to call for making the HTTP request.
:param url: The url to make the request to.
:param raw: Whether to return the response body as JSON or a raw data string (defaults to JSON).
:return: The JSON dictionary from any 200 or 201 responses, or None if a 204 was encountered.
:raise fetchcore.exceptions.RequestException: Corresponding subclass thrown for associated HTTP response error.
"""
try:
response = action(url, timeout=self.TIMEOUT, **kwargs)
except requests.exceptions.RequestException as e:
raise exceptions.ConnectionError(
'Encountered error while making request to %s, %s', url, e)
# 200 OKAY or 201 CREATED have a JSON response
if response.status_code == 200 or response.status_code == 201:
return response.content if raw else response.json()
# 204 NO CONTENT is okay, but we have nothing to parse
elif response.status_code == 204:
return None
elif response.status_code == 400:
self.__log_response_warning(response)
raise exceptions.BadRequest(response.text, response.url,
response.status_code, response.json())
elif response.status_code == 401:
self.__log_response_warning(response)
raise exceptions.Unauthorized(
response.json().get('detail', response.text), response.url,
response.status_code, response.json())
elif response.status_code == 403:
self.__log_response_warning(response)
raise exceptions.Forbidden(
response.json().get('detail', response.text), response.url,
response.status_code, response.json())
elif response.status_code == 404:
self.__log_response_warning(response)
raise exceptions.NotFound("%s Not Found" % response.url,
response.url, response.status_code)
elif response.status_code == 405:
self.__log_response_warning(response)
raise exceptions.MethodNotAllowed(
response.json().get('detail', response.text), response.url,
response.status_code, response.json())
elif response.status_code == 409:
self.__log_response_warning(response)
# TODO: verify 409 returns a detail (we have yet to find something that returns a 409)
raise exceptions.Conflict(
response.json().get('detail', response.text), response.url,
response.status_code, response.json())
elif response.status_code == 500:
self.__log_response_error(response)
raise exceptions.InternalServerError(response.text, response.url,
response.status_code)
else:
# TODO add other 400 errors if Django even throws them
raise exceptions.RequestException(response.text, response.url,
response.status_code)
def test_call_orthologs(self, orth_mock, HOG_mock):
"""Tests that call_orthologs properly calls oma.get_orthologs if get_HOGs fails"""
HOG_mock.side_effect = exceptions.RequestException(
'There was an issue querying the database. Status code 401')
orth_mock.return_value = self.path + 'atn1seq.orth'
test = self.test_logo.call_orthologs()
self.assertTrue(os.path.isfile(test))
self.assertTrue(HOG_mock.called)
self.assertTrue(orth_mock.called)
def send_file(self, data_type, path, now, suffix=None):
"""
Send a file to the ON service.
Args:
data_type: type of data that is being sent.
path: local file path.
now: the time period that corresponds to the file.
"""
if suffix:
name = '{}_{}'.format(self.ona_name, suffix)
else:
name = self.ona_name
url = '{server}/sign/{type}/{year}/{month}/{day}/{time}/{name}'
url = url.format(server=self.proxy_uri,
type=data_type,
year=now.year,
month=now.month,
day=now.day,
time=now.time(),
name=name)
logging.info('Prepping file: {}'.format(url))
sign_headers = {}
if self.sensor_ext_only:
sign_headers = {'sensor-ext-only': 'true'}
response = requests.get(url, headers=sign_headers, **self.request_args)
response.raise_for_status()
result = response.json()
try:
headers = result['headers']
url = result['url']
method = result['method']
remote_path = result['path']
except KeyError:
raise requests_exceptions.RequestException(
'Parameters missing from response')
with io.open(path, mode='rb') as data:
logging.info('Sending file: {} {}'.format(method, url))
resp = requests.request(
method,
url,
headers=headers,
data=data,
verify=True,
timeout=HTTP_TIMEOUT,
)
if resp.status_code == REQUEST_ENTITY_TOO_LARGE:
logging.error('requests error: payload too large')
return None
resp.raise_for_status()
return remote_path
def send(self, request, **kwargs):
if_match = request.headers.get('if-match')
if if_match and if_match != MockAdapter._ETAG:
response = Response()
response._content = request.body
response.headers = {'ETag': MockAdapter._ETAG}
raise exceptions.RequestException(response=response)
resp = super(MockAdapter, self).send(request, **kwargs)
resp.headers = {'ETag': MockAdapter._ETAG}
return resp
def test_call_orthologs_except(self, orth_mock, HOG_mock):
"""Tests that call_orthologs properly calls oma.get_orthologs if get_HOGs fails"""
HOG_mock.side_effect = exceptions.RequestException(
'There was an issue querying the database. Status code 401')
orth_mock.return_value = self.ex_seq
test = self.CDC48A.call_orthologs()
self.assertTrue(os.path.isfile(test))
self.assertTrue(HOG_mock.called)
self.assertTrue(orth_mock.called)
self.assertTrue('Protein_Sequence.orth' in test)
async def async_request(self):
if self.url is None:
self.url = APIStaticV4.BASE_URL
async with ClientSession(headers=self.headers) as session:
tries = 1
while tries <= 3:
if self.query is not None:
if self.query_params is None:
response = await session.post(
url=self.url, json=dict(query=self.query))
else:
response = await session.post(
url=self.url,
json=dict(query=self.query.format_map(
self.query_params)))
else:
response = await session.get(url=self.url)
try:
response.raise_for_status()
except ClientResponseError as e:
# TODO: Raise GrasRequestException
raise e
if response.status == 200:
content = await response.json()
if "errors" in content:
raise exceptions.RequestException(
f"Problem with getting data via url {self.url} + "
f"{self.query.format_map(self.query_params)}.")
return content
else:
logging.error(
f"Problem with getting data via url {self.url}. Error: {response.text}"
)
tries += 1
raise exceptions.RequestException(
f"Problem with getting data via url {self.url} + "
f"{self.query.format_map(self.query_params)}.")
def test_k8s_get_discovery_url_fail(self, mock_get):
CONF.set_override('etcd_discovery_service_endpoint_format',
'http://etcd/test?size=%(size)d',
group='cluster')
mock_get.side_effect = req_exceptions.RequestException()
mock_cluster = mock.MagicMock()
mock_cluster.master_count = 10
mock_cluster.discovery_url = None
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
self.assertRaises(exception.GetDiscoveryUrlFailed,
k8s_def.get_discovery_url, mock_cluster)
def post(self, data):
headers = {"Content-Type": "application/xml; charset=UTF-8"}
if self.cert and self.key:
response = requests.post(url=self.url,
data=data,
headers=headers,
cert=(self.cert, self.key))
else:
raise exceptions.RequestException(
errno="1",
strerror="Please provide cert and key.",
)
return response
def test_modify_ca_unavailable(self):
order_meta = mock.ANY
plugin_meta = {dogtag_import.DogtagCAPlugin.REQUEST_ID:
self.request_id_mock}
self.certclient_mock.review_request.side_effect = (
request_exceptions.RequestException("request_exception"))
result_dto = self.plugin.modify_certificate_request(
self.order_id, order_meta, plugin_meta, self.barbican_meta_dto)
self.assertEqual(cm.CertificateStatus.CA_UNAVAILABLE_FOR_REQUEST,
result_dto.status,
"result_dto_status incorrect")
def post_message(self, payload: Dict) -> bool: # pragma: no cover
response = post(
self._hook_url,
json=payload,
headers=self._headers,
proxies=self._proxy,
timeout=self._timeout,
verify=self._verify,
)
if response.status_code == codes.ok and response.text == "1": # pylint: disable=no-member
return True
else:
raise exceptions.RequestException(response.text)
def send_to_one(peer, path, message=""):
"""
Send a message to a particular node
Arguments:
----------
- `peer`: the node to send the message
- `path`: message to send
- `sender`: adress of the sender
"""
url = "http://{}/{}".format(peer, path)
response = get(url, params=message, timeout=10)
if response.status_code != 200:
raise exceptions.RequestException('Bad return error')
return response
def _query(self, http, url):
root = GraphqlQuery()
root.add_child_node(self)
in_query = {'query': str(root)}
res = http.post(url, json=in_query)
if res.status_code != requests.codes.ok:
raise exceptions.RequestException({
'response': res,
'request': in_query
})
json = res.json()
if 'errors' in json:
raise GhGraphQLError(json)
root.bind(json)
return res
def test_get_solr_data_when_error(self):
"""
Test when solr returns status_code 2xx vs when there is an error
:return:
"""
bibcodes = [
"2020AAS...23528705A", "2019EPSC...13.1911A",
"2019AAS...23338108A", "2019AAS...23320704A",
"2018EPJWC.18608001A", "2018AAS...23221409A",
"2018AAS...23136217A", "2018AAS...23130709A",
"2017ASPC..512...45A", "2015scop.confE...3A"
]
with mock.patch.object(self.current_app.client, 'get') as get_mock:
get_mock.return_value = mock_response = mock.Mock()
mock_response.raise_for_status = mock.Mock()
mock_response.raise_for_status.side_effect = exceptions.RequestException(
"Malformed request")
mock_response.status_code = 400
solr_data = get_solr_data(
bibcodes=bibcodes,
fields='bibcode,author,year,pub,bibstem',
sort=self.current_app.config['EXPORT_SERVICE_NO_SORT_SOLR'])
self.assertEqual(solr_data, None)
# with status code 200
with mock.patch.object(self.current_app.client, 'get') as get_mock:
get_mock.return_value = mock_response = mock.Mock()
mock_response.json.return_value = solrdata.data_6
mock_response.status_code = 200
solr_data = get_solr_data(
bibcodes=bibcodes,
fields='bibcode,author,year,pub,bibstem',
sort=self.current_app.config['EXPORT_SERVICE_NO_SORT_SOLR'])
self.assertEqual(len(solr_data['response']['docs']), len(bibcodes))
# response 203 is also acceptable, it means response is coming from another solr instance then
# where the service is running in
with mock.patch.object(self.current_app.client, 'get') as get_mock:
get_mock.return_value = mock_response = mock.Mock()
mock_response.status_code = 203
mock_response.json.return_value = solrdata.data_6
solr_data = get_solr_data(
bibcodes=bibcodes,
fields='bibcode,author,year,pub,bibstem',
sort=self.current_app.config['EXPORT_SERVICE_NO_SORT_SOLR'])
self.assertEqual(len(solr_data['response']['docs']), len(bibcodes))
def HOG_to_fasta(self):
"""
Retrieves the fasta file containing the sequences of the proteins in the HOG of the input protein
"""
url = constool.build_url(base_url=self.OMA_BASE_URL,
tail='/oma/hogs/{0}/{1}/fasta/',
variation=[self.id, self.hog_level])
response = requests.get(url)
if response.status_code == 200:
self.HOGs = str(response.text)
return self.HOGs
else:
self.save_status = response.status_code
raise exceptions.RequestException(
'There was an issue querying the database. Status code {0}'.
format(self.save_status))
def test_issue_return_ca_unavailable(self):
order_meta = {dogtag_import.DogtagCAPlugin.PROFILE_ID: self.profile_id}
plugin_meta = {}
self.certclient_mock.enroll_cert.side_effect = (
request_exceptions.RequestException())
result_dto = self.plugin.issue_certificate_request(
self.order_id, order_meta, plugin_meta)
self.certclient_mock.enroll_cert.assert_called_once_with(
self.profile_id, order_meta)
self.assertEqual(result_dto.status,
cm.CertificateStatus.CA_UNAVAILABLE_FOR_REQUEST,
"result_dto status incorrect")
def __init__(self, city_name):
self.name = str(city_name).capitalize()
try:
self._request = requests.get(City._base_url.format(self.name))
except e:
raise exceptions.ConnectionError(e)
if self._request.status_code != 200:
raise exceptions.RequestException(self._request.status_code, self._request.text)
if len(self._request.json()) == 0:
raise NameError('Unknown city name: {}'.format(city_name))
else:
try:
station = next(match for match in self._request.json() if match["category"] == "station")
except StopIteration as e:
raise NameError('Not train station found for {}.\nBest result was {}'.format(self.name, self._request.json()[0]))
self.code = station['id']
self.label = station['label']
def run_query(self, ):
q = self.get_query()
if self.debug:
print 'Query started at: %s' % datetime.now()
try:
req = requests.post(self.url, data=q, stream=True)
CHUNK = 1024 * 1024 * 5 # 5MB chunks
with open(self.raw_osm, 'wb') as fd:
for chunk in req.iter_content(CHUNK):
fd.write(chunk)
except exceptions.RequestException as e:
logger.error('Overpass query threw: {0}'.format(e))
raise exceptions.RequestException(e)
if self.debug:
print 'Query finished at %s' % datetime.now()
print 'Wrote overpass query results to: %s' % self.raw_osm
return self.raw_osm
def test_analyze_exception_request(self):
self.assertEqual(errors.SYSTEM_TRANSIENT,
self.apic_handler.analyze_exception(rexc.Timeout()))
self.assertEqual(
errors.SYSTEM_TRANSIENT,
self.apic_handler.analyze_exception(rexc.ConnectionError()))
self.assertEqual(
errors.OPERATION_CRITICAL,
self.apic_handler.analyze_exception(rexc.InvalidURL()))
self.assertEqual(
errors.OPERATION_CRITICAL,
self.apic_handler.analyze_exception(rexc.URLRequired()))
self.assertEqual(
errors.OPERATION_TRANSIENT,
self.apic_handler.analyze_exception(rexc.RequestException()))
self.assertEqual(errors.UNKNOWN,
self.apic_handler.analyze_exception(Exception()))
def post(self, request_headers, data):
if self.cert is None and self.key is None:
response = pkcs12_post(
self.url, data=data,
headers=request_headers,
pkcs12_data=self.p12_buffer,
pkcs12_password=self.password)
elif self.p12_buffer is None and self.password is None:
response = pkcs12_post(
self.url,
data=data,
headers=request_headers,
cert=(self.cert, self.key))
else:
raise exceptions.RequestException(
1,
'Please provide cert and key, or p12 buffer and its password.')
return response
def update_orthoIDs(self):
"""
Takes the OMA specific ID of a protein species, and returns a list of the
canonical IDs of the orthologs of that protein
Returns:
A list of strings, the canonical IDS for the orthologs of the protein
"""
url = constool.build_url(base_url=self.OMA_BASE_URL,
tail='/api/protein/{0}/orthologs/',
variation=[self.id])
response = requests.get(url, headers=self.HEADERS)
if response.status_code == 200:
self.read_resp_orthoIDs(response)
else:
self.save_status = response.status_code
raise exceptions.RequestException(
'There was an issue querying the database. Status code {0}'.
format(self.save_status))
def run_query(self, ):
"""
Run the overpass query.
Return:
the path to the overpass extract
"""
from ..tasks.export_tasks import update_progress
q = self.get_query()
logger.debug(q)
logger.debug('Query started at: %s'.format(datetime.now()))
try:
req = requests.post(self.url,
data=q,
stream=True,
verify=self.verify_ssl)
# Since the request takes a while, jump progress to an arbitrary 50 percent...
update_progress(self.task_uid, progress=50)
try:
size = int(req.headers.get('content-length'))
except (ValueError, TypeError):
if req.content:
size = len(req.content)
else:
raise Exception("Overpass Query failed to return any data")
inflated_size = size * 2
CHUNK = 1024 * 1024 * 2 # 2MB chunks
with open(self.raw_osm, 'wb') as fd:
for chunk in req.iter_content(CHUNK):
fd.write(chunk)
size += CHUNK
# Because progress is already at 50, we need to make this part start at 50 percent
update_progress(
self.task_uid,
progress=(float(size) / float(inflated_size)) * 100)
except exceptions.RequestException as e:
logger.error('Overpass query threw: {0}'.format(e))
raise exceptions.RequestException(e)
if self.debug:
logger.debug('Query finished at %s'.format(datetime.now()))
logger.debug('Wrote overpass query results to: %s'.format(
self.raw_osm))
return self.raw_osm
def post(self, data):
headers = {'Content-Type': 'application/xml; charset=UTF-8'}
if self.cert is None and self.key is None:
response = pkcs12_post(self.url,
data=data,
headers=headers,
pkcs12_data=self.p12_buffer,
pkcs12_password=self.password)
elif self.p12_buffer is None and self.password is None:
response = pkcs12_post(self.url,
data=data,
headers=headers,
cert=(self.cert, self.key))
else:
raise exceptions.RequestException(
errno='1',
strerror=
'Please provide cert and key, or p12 buffer and its password.')
return response
def ortholog_to_fasta(self):
"""
Takes an OMA specific ID and returns a fasta string of the orthologs assciated
with that protein
Returns:
A single fasta string containing the orthologs of that protein, as
dictated by OMA.Note that when the fasta file is parsed,
the first id is the OMA ID, and the second is the canonical id.
"""
url = constool.build_url(base_url=self.OMA_BASE_URL,
tail='/oma/vps/{0}/fasta/',
variation=[self.id])
response = requests.get(url)
if response.status_code == 200:
self.orthologs = str(response.text)
return self.orthologs
else:
self.save_status = response.status_code
raise exceptions.RequestException(
'There was an issue querying the database. Status code {0}'.
format(self.save_status))
def retrieve_OMAid(self):
"""
Takes a protein sequence and returns the oma id of the best protein
match
Returns:
A string containing the ID of the best protein match for the entered sequence
"""
url = constool.build_url(base_url=self.OMA_BASE_URL,
tail='/api/sequence/?query={0}',
variation=[self.sequence])
response = requests.get(url, headers=self.HEADERS)
if response.status_code == 200:
self.read_resp_protID(response)
if response.status_code == 504:
self.save_status = response.status_code
raise TimeoutError(
'The database timed out. Could not determine the orthologs of your sequence. Status code {0}'
.format(self.save_status))
if response.status_code != 200:
self.save_status = response.status_code
raise exceptions.RequestException(
'There was an issue querying the database. Status code {0}'.
format(self.save_status))
def retrieve_HOG_level(self, root=True):
"""
Retrieve information on the taxonomic levels that the HOG spans through
Args:
root(Boolean): if true, return the deepest level of the HOG. If false, return a list
of the alternative level that the HOG spans through
Returns: The deepest level relating the HOG, or a list of all the levels
"""
url = constool.build_url(base_url=self.OMA_BASE_URL,
tail='/api/hog/{0}/',
variation=[self.id])
response = requests.get(url, headers=self.HEADERS)
if response.status_code == 200:
return self.read_HOGid(response, root)
if response.status_code == 504:
self.save_status = response.status_code
raise TimeoutError(
'The database timed out. Could not determine the orthologs of your sequence. Status code {0}'
.format(self.save_status))
if response.status_code != 200:
self.save_status = response.status_code
raise exceptions.RequestException(
'There was an issue querying the database. Status code {0}'.
format(self.save_status))
def test_child_template_when_fetching_file_fails(self):
urlfetch.get.side_effect = exceptions.RequestException()
t = template_format.parse(self.test_template)
stack = self.parse_stack(t)
nested_stack = stack['the_nested']
self.assertRaises(ValueError, nested_stack.child_template)
def side_effect_failure(url):
raise exceptions.RequestException("No Model")
