示例#1
0
    def test_16_metrics_retries(self):
        """ test_16_metrics_retries
 
            Check that we are doing the correct behaviour for retries.
            Retries for rate limits should go forever, with exponential falloff
            Exceeding retry limit should result in a failure
        """
        mock_provider = ProviderMock(
            metrics_exceptions={
                1:[ProviderRateLimitError,ProviderRateLimitError,ProviderRateLimitError],
                2:[ProviderTimeout,ProviderTimeout,ProviderTimeout,ProviderTimeout, ProviderTimeout, ProviderTimeout],
            }
        ) 
        pmt = ProviderMetricsThread(mock_provider, self.config)
        pmt.queue = QueueMock(max_items=2)
        
        start = time.time()
        pmt.start()
        while (pmt.queue.current_item <= 2): 
            time.sleep(1)
        took = time.time() - start
        pmt.stop()
        pmt.join()

        # Total time should be 3 * exponential backoff, and 3 * constant (linear) delay
        assert took >= (1 + 2 + 4) + (0.1 * 3), took
        # Check that item 1 was processed correctly, after retries
        self.assertTrue(mock_provider.metrics_processed.has_key(1))
示例#2
0
 def _spawn_threads(self):
     
     for provider in self.providers:
         if not provider.provides_metrics:
             continue
         logger.info("Spawning thread for provider " + str(provider.provider_name))
         # create and start the metrics threads
         t = ProviderMetricsThread(provider, self.dao)
         t.start()
         self.threads.append(t)
     
     logger.info("Spawning thread for aliases")
     alias_thread = ProvidersAliasThread(self.providers, self.dao)
     alias_thread.start()
     self.threads.append(alias_thread)
示例#3
0
 def test_12_metrics_stopped(self):
     # relies on Queue.first mock as per setUp
     pmt = ProviderMetricsThread(ProviderMock(), self.d)
     pmt.queue = QueueMock()
     
     pmt.start()
     pmt.stop()
     pmt.join()
     
     # there are no assertions to make here, only that the
     # test completes without error
     assert True
示例#4
0
 def test_13_metrics_running(self):
     # relies on Queue.first mock as per setUp
     pmt = ProviderMetricsThread(ProviderMock(), self.d)
     pmt.queue = QueueMock()
     
     start = time.time()
     pmt.start()
     time.sleep(2)
     
     pmt.stop()
     pmt.join()
     took = time.time() - start
     
     # there are no assertions to make here, only that the
     # test completes without error in the appropriate time
     assert took > 2.0
     assert took < 2.5
示例#5
0
    def test_15_metrics_exceptions(self):
        """ test_15_metrics_exceptions

            Check exceptions raised by the metric function on providers
            This test ensures that we generate and handle each exception
            type possible from the providers.
        """
        mock_provider = ProviderMock(
            metrics_exceptions={
                1:[ProviderTimeout,ProviderTimeout],
                2:[ProviderHttpError],
                3:[ProviderClientError],
                4:[ProviderServerError],
                5:[ProviderTimeout,ProviderRateLimitError],
                6:[ProviderContentMalformedError],
                7:[ProviderValidationFailedError],
                8:[ProviderConfigurationError],
                9:[Exception],
            }
        ) 
        pmt = ProviderMetricsThread(mock_provider, self.config)
        pmt.queue = QueueMock(max_items=10)
        
        pmt.start()
        while (pmt.queue.current_item <= 10): 
            time.sleep(1)
    
        pmt.stop()
        pmt.join()

        # Check that items 1,2 were all processed correctly, after a retry
        self.assertTrue(mock_provider.metrics_processed.has_key(1))
        self.assertTrue(mock_provider.metrics_processed.has_key(2))
    
        # Check that item 9 did not get processed as it had a permanent failure
        self.assertFalse(mock_provider.metrics_processed.has_key(9))

        # Check that item 10 was processed correctly 
        self.assertTrue(mock_provider.metrics_processed.has_key(10))
示例#6
0
def main(logfile=None):

    logger = logging.getLogger()

    mydao = dao.Dao(
        app.config["DB_NAME"],
        app.config["DB_URL"],
        app.config["DB_USERNAME"],
        app.config["DB_PASSWORD"]
    ) 

    # Adding this by handle. fileConfig doesn't allow filters to be added
    from totalimpact.backend import ctxfilter
    handler = logging.handlers.RotatingFileHandler(logfile)
    handler.level = logging.DEBUG
    formatter = logging.Formatter("%(asctime)s %(levelname)8s %(item)8s %(thread)s%(provider)s - %(message)s")#,"%H:%M:%S,%f")
    handler.formatter = formatter
    handler.addFilter(ctxfilter)
    logger.addHandler(handler)
    ctxfilter.threadInit()

    logger.debug("test")

    from totalimpact.backend import TotalImpactBackend, ProviderMetricsThread, ProvidersAliasThread, StoppableThread, QueueConsumer
    from totalimpact.providers.provider import Provider, ProviderFactory

    # Start all of the backend processes
    print "Starting alias retrieval thread"
    providers = ProviderFactory.get_providers(app.config["PROVIDERS"])

    alias_threads = []
    thread_count = app.config["ALIASES"]["workers"]
    for idx in range(thread_count):
        at = ProvidersAliasThread(providers, mydao, idx)
        at.thread_id = 'AliasThread(%i)' % idx
        at.start()
        alias_threads.append(at)

    print "Starting metric retrieval threads..."
    # Start each of the metric providers
    metrics_threads = []
    for provider in providers:
        providers = ProviderFactory.get_providers(app.config["PROVIDERS"])
        thread_count = app.config["PROVIDERS"][provider.provider_name]["workers"]
        print "  ", provider.provider_name
        for idx in range(thread_count):
            thread = ProviderMetricsThread(provider, mydao)
            metrics_threads.append(thread)
            thread.thread_id = thread.thread_id + '(%i)' % idx
            thread.start()

    # Install a signal handler so we'll break out of the main loop
    # on receipt of relevant signals
    class ExitSignal(Exception):
        pass
 
    def kill_handler(signum, frame):
        raise ExitSignal()

    import signal
    signal.signal(signal.SIGTERM, kill_handler)

    try:
        while True:
            time.sleep(1)
    except (KeyboardInterrupt, ExitSignal), e:
        pass
    def test_metrics_queue(self):
        """ Test that the metrics queue works

            This test isn't correct just now. We'd need to simulate
            the item getting it's aliases completed.
        """
        raise SkipTest
        self.d.create_new_db_and_connect(self.testing_db_name)
        number_of_item_api_calls = 0

        # create new dryad item 
        dryad_resp = self.client.post('/item/doi/' + 
                quote_plus(DRYAD_TEST_DOI))
        number_of_item_api_calls += 1
        dryad_tiid = dryad_resp.data

        # test the metrics view works
        res = self.d.view("metrics")
        assert_equals(
            len(res["rows"]),
             number_of_item_api_calls*len(self.providers)
            )  # three IDs above, three providers
        assert_equals(
            res["rows"][0]["value"]["metrics"]["dryad:package_views"]["values"],
            {})

        # see if the item is on the queue
        all_metrics_queue = MetricsQueue(self.d) 
        assert isinstance(all_metrics_queue.queue, list)
        assert_equals(
            len(all_metrics_queue.queue),
            number_of_item_api_calls*len(self.providers)
            )
        
        # get our item from the queue
        my_item = all_metrics_queue.first() 
        assert_equals(my_item.metrics["dryad:package_views"]['values'], {})
        assert(my_item.created - time.time() < 30)


        # create new plos item 
        plos_resp = self.client.post('/item/doi/' + quote_plus(PLOS_TEST_DOI))
        number_of_item_api_calls += 1        
        plos_tiid = json.loads(plos_resp.data)

        # create new github item 
        github_resp = self.client.post('/item/github/' + quote_plus(GITHUB_TEST_ID))
        number_of_item_api_calls += 1        
        github_tiid = json.loads(github_resp.data)

        all_metrics_queue = MetricsQueue(self.d)
        #assert_equals(len(all_metrics_queue.queue), 
        #        number_of_item_api_calls*len(self.providers)) 

        dryad_metrics_queue = MetricsQueue(self.d, "dryad")
        assert_equals(len(dryad_metrics_queue.queue), 
                number_of_item_api_calls) 

        github_metrics_queue = MetricsQueue(self.d, "github")
        assert_equals(len(github_metrics_queue.queue), 
                number_of_item_api_calls) 


        alias_thread = ProvidersAliasThread(self.providers, self.d)
        alias_thread.run(run_only_once=True)

        # now run just the dryad metrics thread.
        metrics_thread = ProviderMetricsThread(self.providers[0], self.d)
        metrics_thread.run(run_only_once=True)  
        metrics_thread.run(run_only_once=True)
        metrics_thread.run(run_only_once=True)

        # test the dryad doi
        dryad_resp = self.client.get('/item/' + dryad_tiid.replace('"', ''))

        resp_dict = json.loads(dryad_resp.data)
        print json.dumps(resp_dict, sort_keys=True, indent=4) 

        assert_equals(resp_dict['metrics']['dryad:total_downloads']['values'].values()[0],
            169)