def should_not_get_into_infinite_recursion_if_lower_limit_is_invalid_and_throw_regular_regquest_exception(
mock_httpconn):
# case where error's lowerlimit for some reason won't work and recursion should be stopped after one
# retry and throw Request exception with error code 400.
mock_connection = mock_httpconn.return_value
mock_connection.getresponse.side_effect = [
MockResponse(400, etree.tostring(out_of_allowed_range_error)),
MockResponse(400, etree.tostring(out_of_allowed_range_error)),
MockResponse(400, etree.tostring(out_of_allowed_range_error))
]
fmi_request = FMIRequest('apikey')
query = create_daily_query(
datetime(2012, 1, 9, hour=2, minute=1, second=0,
microsecond=0),
datetime(2012,
1,
14,
hour=2,
minute=1,
second=0,
microsecond=0))
with pytest.raises(RequestException) as e:
fmi_request.get(query)
# Make sure that the request was retried
assert_equal(2, mock_connection.getresponse.call_count)
# Should raise RequestException with information given by FMIApi in xml
assert_equal(400, e.value.error_code)
assert 'Couldn\'t retrieve data even with found lowerlimit date' in e.value.message
def should_try_to_find_lower_limit_datetime_if_out_of_range_response_encountered_and_try_again_with_it(
mock_httpconn):
# case where server returns out of range exception and we find instead new lower limit to retrieve with.
# in short, first get failure with out of range, then do a new request with success
mock_connection = mock_httpconn.return_value
mock_connection.getresponse.side_effect = [
MockResponse(400, etree.tostring(out_of_allowed_range_error)),
MockResponse(200, etree.tostring(realtime_1_day))
]
fmi_request = FMIRequest('apikey')
query = create_daily_query(
datetime(2012, 1, 9, hour=2, minute=1, second=0,
microsecond=0),
datetime(2012,
1,
14,
hour=2,
minute=1,
second=0,
microsecond=0))
result = fmi_request.get(query)
# Make sure that the request was retried
assert_equal(2, mock_connection.getresponse.call_count)
# Returned dataframe should have all data available in provided xml
parser = FMIxmlParser()
result = parser.parse(result)
verify_dataframe(result, EXPECTED_REALTIME_1_DAY)
def should_raise_general_request_exception_if_lowerlimit_date_cannot_be_parsed(
mock_httpconn):
# case where error's datetime is in invalid or unsupported format
mock_connection = mock_httpconn.return_value
mock_connection.getresponse.side_effect = [
MockResponse(400,
etree.tostring(out_of_allowed_invalid_lowerlimit))
]
fmi_request = FMIRequest('apikey')
query = create_daily_query(
datetime(2012, 1, 9, hour=2, minute=1, second=0,
microsecond=0),
datetime(2012,
1,
14,
hour=2,
minute=1,
second=0,
microsecond=0))
with pytest.raises(RequestException) as e:
fmi_request.get(query)
# Should raise RequestException with information given by FMIApi in xml
assert_equal(400, e.value.error_code)
assert 'value 2012-Tam-13 00:00:00 is out of allowed range' in e.value.message
def should_raise_general_request_exception_if_lowerlimit_cannot_be_found(
mock_httpconn):
# case where error won't contain lowerlimit and regular exception should be thrown
mock_connection = mock_httpconn.return_value
mock_connection.getresponse.side_effect = [
MockResponse(400, etree.tostring(out_of_allowed_no_lowerlimit))
]
fmi_request = FMIRequest('apikey')
query = create_daily_query(
datetime(2012, 1, 9, hour=2, minute=1, second=0,
microsecond=0),
datetime(2012,
1,
14,
hour=2,
minute=1,
second=0,
microsecond=0))
with pytest.raises(RequestException) as e:
fmi_request.get(query)
# Should raise RequestException with information given by FMIApi in xml
assert_equal(400, e.value.error_code)
assert 'This is just a random error' in e.value.message
def should_do_request_to_fmi_with_provided_query_params_and_return_data(
mock_httpconn):
mock_connection = mock_httpconn.return_value
mock_connection.getresponse.return_value = MockResponse(
200, '<asd>Important data</asd>')
fmi_request = FMIRequest('apikey')
query = create_daily_query(
datetime(2010, 1, 1, hour=2, minute=1, second=0, microsecond=0),
datetime(2011, 1, 5, hour=2, minute=1, second=0, microsecond=0))
result = fmi_request.get(query)
assert_equal(1, mock_connection.getresponse.call_count)
assert_equal('Important data', result.text)
def should_get_exception_reason_from_html_response(mock_httpconn):
mock_connection = mock_httpconn.return_value
mock_connection.getresponse.return_value = MockResponse(
404, html_error.encode('utf-8'), content_type='text/html')
fmi_request = FMIRequest('apikey')
query = create_daily_query(
datetime(2010, 1, 1, hour=0, minute=1, second=0, microsecond=0),
datetime(2011, 1, 5, hour=0, minute=1, second=0, microsecond=0))
with pytest.raises(RequestException) as e:
fmi_request.get(query)
assert_equal(404, e.value.error_code)
assert 'Some module: Some random error' in e.value.html
def should_get_exception_reason_from_xml_response(mock_httpconn):
mock_connection = mock_httpconn.return_value
mock_connection.getresponse.return_value = MockResponse(
400,
etree.tostring(xml_error),
content_type='text/xml; charset=UTF8')
fmi_request = FMIRequest('apikey')
query = create_daily_query(
datetime(1916, 5, 23, hour=21, minute=31, second=57,
microsecond=0),
datetime(2011, 5, 27, hour=21, minute=31, second=57,
microsecond=0))
with pytest.raises(RequestException) as e:
fmi_request.get(query)
assert_equal(400, e.value.error_code)
assert 'Too long time interval \'1916-May-23 21:31:57\' to \'2011-May-27 21:31:57\'' in e.value.message
class FMIRequestHandler:
"""
This class takes a data request and splits it to multiple http-requests if required and
does the request by using FMIRequest class.
"""
def __init__(self, api_key):
self._api_key = api_key
self._FMI_request = FMIRequest(self._api_key)
self._callbackFunction = None
def request(self, params, max_timespan, progress_callback=None):
requests = self._divide_to_multiple_requests(params, max_timespan)
return self._execute_requests(requests, progress_callback)
def _execute_requests(self, requests, progress_callback):
all_requests = len(requests)
responses = []
for i, r in enumerate(requests):
try:
responses.append(self._do_request(r))
if progress_callback is not None:
progress_callback(i, all_requests)
except RequestException as e:
# If result is 400, hope that the next request in batch will work. Raise other errors normally.
# Handles case where beginning of a multipart request won't contain data
# FIXME: Could be done in a way where after new lowerlimit is found, a new batch of requests is calculated instead of doing
# FIXME: bunch of useless requests.
print('Exception on request', e)
if e.error_code != 400:
raise e
if progress_callback is not None:
progress_callback(i, all_requests)
return responses
def _do_request(self, request):
return self._FMI_request.get(request)
@staticmethod
def _divide_to_multiple_requests(params, max_timespan):
requests = []
done = False
i = 0
while not done:
request_params = copy.copy(params)
request_params["starttime"] += datetime.timedelta(
hours=max_timespan) * i
request_params["endtime"] = request_params[
"starttime"] + datetime.timedelta(hours=max_timespan)
# This additional minute to starting time is to prevent requests from fetching same time twice in
# the splitting point. Otherwise previous request's last time will be fetched as first in the next.
# FMI's service recognizes minutes as smallest significate time step so seconds or milliseconds could not
# be used.
if i > 0:
request_params["starttime"] += datetime.timedelta(minutes=1)
requests.append(request_params)
if request_params["endtime"] > params["endtime"]:
done = True
request_params["endtime"] = params["endtime"]
i += 1
return requests
def __init__(self, api_key):
self._api_key = api_key
self._FMI_request = FMIRequest(self._api_key)
self._callbackFunction = None