def test_16_metrics_retries(self):
""" test_16_metrics_retries
Check that we are doing the correct behaviour for retries.
Retries for rate limits should go forever, with exponential falloff
Exceeding retry limit should result in a failure
"""
mock_provider = ProviderMock(
metrics_exceptions={
1:[ProviderRateLimitError,ProviderRateLimitError,ProviderRateLimitError],
2:[ProviderTimeout,ProviderTimeout,ProviderTimeout,ProviderTimeout, ProviderTimeout, ProviderTimeout],
}
)
pmt = ProviderMetricsThread(mock_provider, self.config)
pmt.queue = QueueMock(max_items=2)
start = time.time()
pmt.start()
while (pmt.queue.current_item <= 2):
time.sleep(1)
took = time.time() - start
pmt.stop()
pmt.join()
# Total time should be 3 * exponential backoff, and 3 * constant (linear) delay
assert took >= (1 + 2 + 4) + (0.1 * 3), took
# Check that item 1 was processed correctly, after retries
self.assertTrue(mock_provider.metrics_processed.has_key(1))
def test_12_metrics_stopped(self):
# relies on Queue.first mock as per setUp
pmt = ProviderMetricsThread(ProviderMock(), self.d)
pmt.queue = QueueMock()
pmt.start()
pmt.stop()
pmt.join()
# there are no assertions to make here, only that the
# test completes without error
assert True
def test_13_metrics_running(self):
# relies on Queue.first mock as per setUp
pmt = ProviderMetricsThread(ProviderMock(), self.d)
pmt.queue = QueueMock()
start = time.time()
pmt.start()
time.sleep(2)
pmt.stop()
pmt.join()
took = time.time() - start
# there are no assertions to make here, only that the
# test completes without error in the appropriate time
assert took > 2.0
assert took < 2.5
def test_15_metrics_exceptions(self):
""" test_15_metrics_exceptions
Check exceptions raised by the metric function on providers
This test ensures that we generate and handle each exception
type possible from the providers.
"""
mock_provider = ProviderMock(
metrics_exceptions={
1:[ProviderTimeout,ProviderTimeout],
2:[ProviderHttpError],
3:[ProviderClientError],
4:[ProviderServerError],
5:[ProviderTimeout,ProviderRateLimitError],
6:[ProviderContentMalformedError],
7:[ProviderValidationFailedError],
8:[ProviderConfigurationError],
9:[Exception],
}
)
pmt = ProviderMetricsThread(mock_provider, self.config)
pmt.queue = QueueMock(max_items=10)
pmt.start()
while (pmt.queue.current_item <= 10):
time.sleep(1)
pmt.stop()
pmt.join()
# Check that items 1,2 were all processed correctly, after a retry
self.assertTrue(mock_provider.metrics_processed.has_key(1))
self.assertTrue(mock_provider.metrics_processed.has_key(2))
# Check that item 9 did not get processed as it had a permanent failure
self.assertFalse(mock_provider.metrics_processed.has_key(9))
# Check that item 10 was processed correctly
self.assertTrue(mock_provider.metrics_processed.has_key(10))
