def test_16_metrics_retries(self): """ test_16_metrics_retries Check that we are doing the correct behaviour for retries. Retries for rate limits should go forever, with exponential falloff Exceeding retry limit should result in a failure """ mock_provider = ProviderMock( metrics_exceptions={ 1:[ProviderRateLimitError,ProviderRateLimitError,ProviderRateLimitError], 2:[ProviderTimeout,ProviderTimeout,ProviderTimeout,ProviderTimeout, ProviderTimeout, ProviderTimeout], } ) pmt = ProviderMetricsThread(mock_provider, self.config) pmt.queue = QueueMock(max_items=2) start = time.time() pmt.start() while (pmt.queue.current_item <= 2): time.sleep(1) took = time.time() - start pmt.stop() pmt.join() # Total time should be 3 * exponential backoff, and 3 * constant (linear) delay assert took >= (1 + 2 + 4) + (0.1 * 3), took # Check that item 1 was processed correctly, after retries self.assertTrue(mock_provider.metrics_processed.has_key(1))
def _spawn_threads(self): for provider in self.providers: if not provider.provides_metrics: continue logger.info("Spawning thread for provider " + str(provider.provider_name)) # create and start the metrics threads t = ProviderMetricsThread(provider, self.dao) t.start() self.threads.append(t) logger.info("Spawning thread for aliases") alias_thread = ProvidersAliasThread(self.providers, self.dao) alias_thread.start() self.threads.append(alias_thread)
def test_12_metrics_stopped(self): # relies on Queue.first mock as per setUp pmt = ProviderMetricsThread(ProviderMock(), self.d) pmt.queue = QueueMock() pmt.start() pmt.stop() pmt.join() # there are no assertions to make here, only that the # test completes without error assert True
def test_13_metrics_running(self): # relies on Queue.first mock as per setUp pmt = ProviderMetricsThread(ProviderMock(), self.d) pmt.queue = QueueMock() start = time.time() pmt.start() time.sleep(2) pmt.stop() pmt.join() took = time.time() - start # there are no assertions to make here, only that the # test completes without error in the appropriate time assert took > 2.0 assert took < 2.5
def test_15_metrics_exceptions(self): """ test_15_metrics_exceptions Check exceptions raised by the metric function on providers This test ensures that we generate and handle each exception type possible from the providers. """ mock_provider = ProviderMock( metrics_exceptions={ 1:[ProviderTimeout,ProviderTimeout], 2:[ProviderHttpError], 3:[ProviderClientError], 4:[ProviderServerError], 5:[ProviderTimeout,ProviderRateLimitError], 6:[ProviderContentMalformedError], 7:[ProviderValidationFailedError], 8:[ProviderConfigurationError], 9:[Exception], } ) pmt = ProviderMetricsThread(mock_provider, self.config) pmt.queue = QueueMock(max_items=10) pmt.start() while (pmt.queue.current_item <= 10): time.sleep(1) pmt.stop() pmt.join() # Check that items 1,2 were all processed correctly, after a retry self.assertTrue(mock_provider.metrics_processed.has_key(1)) self.assertTrue(mock_provider.metrics_processed.has_key(2)) # Check that item 9 did not get processed as it had a permanent failure self.assertFalse(mock_provider.metrics_processed.has_key(9)) # Check that item 10 was processed correctly self.assertTrue(mock_provider.metrics_processed.has_key(10))
def main(logfile=None): logger = logging.getLogger() mydao = dao.Dao( app.config["DB_NAME"], app.config["DB_URL"], app.config["DB_USERNAME"], app.config["DB_PASSWORD"] ) # Adding this by handle. fileConfig doesn't allow filters to be added from totalimpact.backend import ctxfilter handler = logging.handlers.RotatingFileHandler(logfile) handler.level = logging.DEBUG formatter = logging.Formatter("%(asctime)s %(levelname)8s %(item)8s %(thread)s%(provider)s - %(message)s")#,"%H:%M:%S,%f") handler.formatter = formatter handler.addFilter(ctxfilter) logger.addHandler(handler) ctxfilter.threadInit() logger.debug("test") from totalimpact.backend import TotalImpactBackend, ProviderMetricsThread, ProvidersAliasThread, StoppableThread, QueueConsumer from totalimpact.providers.provider import Provider, ProviderFactory # Start all of the backend processes print "Starting alias retrieval thread" providers = ProviderFactory.get_providers(app.config["PROVIDERS"]) alias_threads = [] thread_count = app.config["ALIASES"]["workers"] for idx in range(thread_count): at = ProvidersAliasThread(providers, mydao, idx) at.thread_id = 'AliasThread(%i)' % idx at.start() alias_threads.append(at) print "Starting metric retrieval threads..." # Start each of the metric providers metrics_threads = [] for provider in providers: providers = ProviderFactory.get_providers(app.config["PROVIDERS"]) thread_count = app.config["PROVIDERS"][provider.provider_name]["workers"] print " ", provider.provider_name for idx in range(thread_count): thread = ProviderMetricsThread(provider, mydao) metrics_threads.append(thread) thread.thread_id = thread.thread_id + '(%i)' % idx thread.start() # Install a signal handler so we'll break out of the main loop # on receipt of relevant signals class ExitSignal(Exception): pass def kill_handler(signum, frame): raise ExitSignal() import signal signal.signal(signal.SIGTERM, kill_handler) try: while True: time.sleep(1) except (KeyboardInterrupt, ExitSignal), e: pass
def test_metrics_queue(self): """ Test that the metrics queue works This test isn't correct just now. We'd need to simulate the item getting it's aliases completed. """ raise SkipTest self.d.create_new_db_and_connect(self.testing_db_name) number_of_item_api_calls = 0 # create new dryad item dryad_resp = self.client.post('/item/doi/' + quote_plus(DRYAD_TEST_DOI)) number_of_item_api_calls += 1 dryad_tiid = dryad_resp.data # test the metrics view works res = self.d.view("metrics") assert_equals( len(res["rows"]), number_of_item_api_calls*len(self.providers) ) # three IDs above, three providers assert_equals( res["rows"][0]["value"]["metrics"]["dryad:package_views"]["values"], {}) # see if the item is on the queue all_metrics_queue = MetricsQueue(self.d) assert isinstance(all_metrics_queue.queue, list) assert_equals( len(all_metrics_queue.queue), number_of_item_api_calls*len(self.providers) ) # get our item from the queue my_item = all_metrics_queue.first() assert_equals(my_item.metrics["dryad:package_views"]['values'], {}) assert(my_item.created - time.time() < 30) # create new plos item plos_resp = self.client.post('/item/doi/' + quote_plus(PLOS_TEST_DOI)) number_of_item_api_calls += 1 plos_tiid = json.loads(plos_resp.data) # create new github item github_resp = self.client.post('/item/github/' + quote_plus(GITHUB_TEST_ID)) number_of_item_api_calls += 1 github_tiid = json.loads(github_resp.data) all_metrics_queue = MetricsQueue(self.d) #assert_equals(len(all_metrics_queue.queue), # number_of_item_api_calls*len(self.providers)) dryad_metrics_queue = MetricsQueue(self.d, "dryad") assert_equals(len(dryad_metrics_queue.queue), number_of_item_api_calls) github_metrics_queue = MetricsQueue(self.d, "github") assert_equals(len(github_metrics_queue.queue), number_of_item_api_calls) alias_thread = ProvidersAliasThread(self.providers, self.d) alias_thread.run(run_only_once=True) # now run just the dryad metrics thread. metrics_thread = ProviderMetricsThread(self.providers[0], self.d) metrics_thread.run(run_only_once=True) metrics_thread.run(run_only_once=True) metrics_thread.run(run_only_once=True) # test the dryad doi dryad_resp = self.client.get('/item/' + dryad_tiid.replace('"', '')) resp_dict = json.loads(dryad_resp.data) print json.dumps(resp_dict, sort_keys=True, indent=4) assert_equals(resp_dict['metrics']['dryad:total_downloads']['values'].values()[0], 169)