def setUp(self, s):
self.lrm = LogRetryMiddleware(MagicMock())
self.lrm.settings = {}
self.lrm.name = 'OverrideSpider'
self.lrm.redis_conn = MagicMock()
self.lrm.logger = MagicMock()
self.lrm.settings['STATS_CYCLE'] = 5
self.lrm.settings['STATS_TIMES'] = []
self.lrm._get_hostname = MagicMock(return_value='host1')
class TestLogRetryMiddlewareStats(TestCase):
@mock.patch('crawling.log_retry_middleware.LogRetryMiddleware' \
'.setup')
def setUp(self, s):
self.lrm = LogRetryMiddleware(MagicMock())
self.lrm.settings = {}
self.lrm.name = 'OverrideSpider'
self.lrm.redis_conn = MagicMock()
self.lrm.logger = MagicMock()
self.lrm.settings['STATS_CYCLE'] = 5
self.lrm.settings['STATS_TIMES'] = []
self.lrm._get_hostname = MagicMock(return_value='host1')
def test_lrm_stats_setup(self):
self.lrm.stats_dict = {}
# test nothing
self.lrm._setup_stats_status_codes()
self.assertEquals(self.lrm.stats_dict.keys(), ['lifetime'])
# test good/bad rolling stats
self.lrm.stats_dict = {}
self.lrm.settings['STATS_TIMES'] = [
'SECONDS_15_MINUTE',
'SECONDS_1_HOUR',
'SECONDS_DUMB',
]
good = [
'lifetime', # for totals, not DUMB
900,
3600,
]
# check that both keys are set up
self.lrm._setup_stats_status_codes()
self.assertEquals(
sorted(self.lrm.stats_dict.keys()),
sorted(good))
k1 = 'stats:crawler:host1:OverrideSpider:504'
for time_key in self.lrm.stats_dict:
if time_key == 0:
self.assertEquals(
self.lrm.stats_dict[0].key,
'{k}:lifetime'.format(k=k1)
)
else:
self.assertEquals(
self.lrm.stats_dict[time_key].key,
'{k}:{t}'.format(k=k1, t=time_key)
)
def setUp(self, s):
self.lrm = LogRetryMiddleware(MagicMock())
self.lrm.settings = {}
self.lrm.name = 'OverrideSpider'
self.lrm.redis_conn = MagicMock()
self.lrm.logger = MagicMock()
self.lrm.settings['STATS_CYCLE'] = 5
self.lrm.settings['STATS_TIMES'] = []
self.lrm._get_hostname = MagicMock(return_value='host1')