def test_16_metrics_retries(self):
""" test_16_metrics_retries
Check that we are doing the correct behaviour for retries.
Retries for rate limits should go forever, with exponential falloff
Exceeding retry limit should result in a failure
"""
mock_provider = ProviderMock(
metrics_exceptions={
1:[ProviderRateLimitError,ProviderRateLimitError,ProviderRateLimitError],
2:[ProviderTimeout,ProviderTimeout,ProviderTimeout,ProviderTimeout, ProviderTimeout, ProviderTimeout],
}
)
pmt = ProviderMetricsThread(mock_provider, self.config)
pmt.queue = QueueMock(max_items=2)
start = time.time()
pmt.start()
while (pmt.queue.current_item <= 2):
time.sleep(1)
took = time.time() - start
pmt.stop()
pmt.join()
# Total time should be 3 * exponential backoff, and 3 * constant (linear) delay
assert took >= (1 + 2 + 4) + (0.1 * 3), took
# Check that item 1 was processed correctly, after retries
self.assertTrue(mock_provider.metrics_processed.has_key(1))
def test_12_metrics_stopped(self):
# relies on Queue.first mock as per setUp
pmt = ProviderMetricsThread(ProviderMock(), self.d)
pmt.queue = QueueMock()
pmt.start()
pmt.stop()
pmt.join()
# there are no assertions to make here, only that the
# test completes without error
assert True
def _spawn_threads(self):
for provider in self.providers:
if not provider.provides_metrics:
continue
logger.info("Spawning thread for provider " + str(provider.provider_name))
# create and start the metrics threads
t = ProviderMetricsThread(provider, self.dao)
t.start()
self.threads.append(t)
logger.info("Spawning thread for aliases")
alias_thread = ProvidersAliasThread(self.providers, self.dao)
alias_thread.start()
self.threads.append(alias_thread)
def test_13_metrics_running(self):
# relies on Queue.first mock as per setUp
pmt = ProviderMetricsThread(ProviderMock(), self.d)
pmt.queue = QueueMock()
start = time.time()
pmt.start()
time.sleep(2)
pmt.stop()
pmt.join()
took = time.time() - start
# there are no assertions to make here, only that the
# test completes without error in the appropriate time
assert took > 2.0
assert took < 2.5
def test_15_metrics_exceptions(self):
""" test_15_metrics_exceptions
Check exceptions raised by the metric function on providers
This test ensures that we generate and handle each exception
type possible from the providers.
"""
mock_provider = ProviderMock(
metrics_exceptions={
1:[ProviderTimeout,ProviderTimeout],
2:[ProviderHttpError],
3:[ProviderClientError],
4:[ProviderServerError],
5:[ProviderTimeout,ProviderRateLimitError],
6:[ProviderContentMalformedError],
7:[ProviderValidationFailedError],
8:[ProviderConfigurationError],
9:[Exception],
}
)
pmt = ProviderMetricsThread(mock_provider, self.config)
pmt.queue = QueueMock(max_items=10)
pmt.start()
while (pmt.queue.current_item <= 10):
time.sleep(1)
pmt.stop()
pmt.join()
# Check that items 1,2 were all processed correctly, after a retry
self.assertTrue(mock_provider.metrics_processed.has_key(1))
self.assertTrue(mock_provider.metrics_processed.has_key(2))
# Check that item 9 did not get processed as it had a permanent failure
self.assertFalse(mock_provider.metrics_processed.has_key(9))
# Check that item 10 was processed correctly
self.assertTrue(mock_provider.metrics_processed.has_key(10))
def main(logfile=None):
logger = logging.getLogger()
mydao = dao.Dao(
app.config["DB_NAME"],
app.config["DB_URL"],
app.config["DB_USERNAME"],
app.config["DB_PASSWORD"]
)
# Adding this by handle. fileConfig doesn't allow filters to be added
from totalimpact.backend import ctxfilter
handler = logging.handlers.RotatingFileHandler(logfile)
handler.level = logging.DEBUG
formatter = logging.Formatter("%(asctime)s %(levelname)8s %(item)8s %(thread)s%(provider)s - %(message)s")#,"%H:%M:%S,%f")
handler.formatter = formatter
handler.addFilter(ctxfilter)
logger.addHandler(handler)
ctxfilter.threadInit()
logger.debug("test")
from totalimpact.backend import TotalImpactBackend, ProviderMetricsThread, ProvidersAliasThread, StoppableThread, QueueConsumer
from totalimpact.providers.provider import Provider, ProviderFactory
# Start all of the backend processes
print "Starting alias retrieval thread"
providers = ProviderFactory.get_providers(app.config["PROVIDERS"])
alias_threads = []
thread_count = app.config["ALIASES"]["workers"]
for idx in range(thread_count):
at = ProvidersAliasThread(providers, mydao, idx)
at.thread_id = 'AliasThread(%i)' % idx
at.start()
alias_threads.append(at)
print "Starting metric retrieval threads..."
# Start each of the metric providers
metrics_threads = []
for provider in providers:
providers = ProviderFactory.get_providers(app.config["PROVIDERS"])
thread_count = app.config["PROVIDERS"][provider.provider_name]["workers"]
print " ", provider.provider_name
for idx in range(thread_count):
thread = ProviderMetricsThread(provider, mydao)
metrics_threads.append(thread)
thread.thread_id = thread.thread_id + '(%i)' % idx
thread.start()
# Install a signal handler so we'll break out of the main loop
# on receipt of relevant signals
class ExitSignal(Exception):
pass
def kill_handler(signum, frame):
raise ExitSignal()
import signal
signal.signal(signal.SIGTERM, kill_handler)
try:
while True:
time.sleep(1)
except (KeyboardInterrupt, ExitSignal), e:
pass
