def test_process_request(self): req = Request('http://scrapytest.org/') self.mw.process_request(req, self.spider) sock_check(self.client._sock, 1, 'udp', val='{}.spider_reqs_issued:1|c'.format(self.statsd_prefix))
def test_process_spider_exception(self): test_exception = IOError() resp = Response('http://scrapytest.org/') self.mw.process_exception(resp, test_exception, self.spider) sock_check(self.client._sock, 1, 'udp', val='{}.error_IOError:1|c'.format(self.statsd_prefix))
def test_process_exception(self): req = Request('http://scrapytest.org/') test_exception = KeyError() self.mw.process_exception(req, test_exception, self.spider) sock_check(self.client._sock, 1, 'udp', val='{}.error_KeyError:1|c'.format(self.statsd_prefix))
def test_process_response(self): req = Request('http://scrapytest.org/') resp = Response('http://scrapytest.org/') self.mw.process_response(req, resp, self.spider) sock_check(self.client._sock, 1, 'udp', val='{}.spider_resps_received:1|c'.format(self.statsd_prefix))
def test_process_spider_output(self): results = [Banana(), Banana(), Apple(), Orange()] resp = Response('http://scrapytest.org/') client = udp_client(self.statsd_prefix) pipeline = client.pipeline() self.mw.statsd_client = pipeline self.mw.process_spider_output(resp, results, self.spider) pipeline.send() expected_val = "\n".join(map(lambda x: "{}.processed_{}:1|c".format(self.statsd_prefix, type(x).__name__), results)) sock_check(client._sock, 1, 'udp', val=expected_val)
def test_init_spider_custom_name(self): hostname = 'captn-fluffington' spider_name = 'test-spider' prefix_pattern = '{hostname}_{name}_dev_' expected_prefix = prefix_pattern.format(hostname=hostname, name=spider_name) settings = {'STATSD_HOSTNAME': hostname, 'STATSD_PREFIX': prefix_pattern} crawler = CrawlerRunner(settings).create_crawler(Spider) spider = crawler._create_spider(spider_name) mw = StatsdMiddleware().from_crawler(crawler) mw.spider_opened(spider) mw.statsd_client = udp_client(expected_prefix) self.assertEqual(mw.statsd_prefix, expected_prefix) req = Request('http://scrapytest.org/') mw.process_request(req, spider) sock_check(mw.statsd_client._sock, 1, 'udp', val='{}.spider_reqs_issued:1|c'.format(expected_prefix))
def test_statsd_client(self): self.mw.statsd_client.incr('foo') sock_check(self.client._sock, 1, 'udp', val='{}.foo:1|c'.format(self.statsd_prefix))