class TestStopPlugin(TestCase, RegexFixer):
def setUp(self):
self.plugin = StopMonitor()
self.plugin.redis_conn = MagicMock()
self.plugin.logger = MagicMock()
def test_stop_regex(self):
regex = self.fix_re(self.plugin.regex)
self.assertEquals(re.findall(regex, "stop:spider:app:crawl"), ["stop:spider:app:crawl"])
self.assertEquals(re.findall(regex, "stop:spider:app"), ["stop:spider:app"])
self.assertEquals(re.findall(regex, "stop:stuff"), [])
def test_stop_monitor_mini_purge(self):
self.plugin.redis_conn.scan_iter = MagicMock(return_value=["link:istresearch.com:queue"])
self.plugin.redis_conn.zscan_iter = MagicMock(
return_value=[
["(dp0\nS'crawlid'\np1\nS'crawl'\np2\nsS'appid'\np3\nS'app'\np4\ns."],
["(dp0\nS'crawlid'\np1\nS'crawl'\np2\nsS'appid'\np3\nS'foo'\np4\ns."],
]
)
self.assertEquals(self.plugin._mini_purge("link", "app", "crawl"), 1)
class TestStopPlugin(TestCase, RegexFixer):
def setUp(self):
self.plugin = StopMonitor()
self.plugin.redis_conn = MagicMock()
self.plugin.logger = MagicMock()
def test_stop_regex(self):
regex = self.fix_re(self.plugin.regex)
self.assertEquals(re.findall(regex, 'stop:spider:app:crawl'),
['stop:spider:app:crawl'])
self.assertEquals(re.findall(regex, 'stop:spider:app'),
['stop:spider:app'])
self.assertEquals(re.findall(regex, 'stop:stuff'), [])
def test_stop_monitor_mini_purge(self):
self.plugin.redis_conn.scan_iter = MagicMock(return_value=['link:istresearch.com:queue'])
self.plugin.redis_conn.zscan_iter = MagicMock(return_value=[
["(dp0\nS'crawlid'\np1\nS'crawl'\np2\nsS'appid'\np3\nS'app'\np4\ns."],
["(dp0\nS'crawlid'\np1\nS'crawl'\np2\nsS'appid'\np3\nS'foo'\np4\ns."],
])
self.assertEquals(self.plugin._mini_purge("link", "app", "crawl"), 1)
class TestStopPlugin(TestCase, RegexFixer):
def setUp(self):
self.plugin = StopMonitor()
self.plugin.redis_conn = MagicMock()
self.plugin.logger = MagicMock()
def test_stop_regex(self):
regex = self.fix_re(self.plugin.regex)
self.assertEquals(re.findall(regex, 'stop:spider:app:crawl'),
['stop:spider:app:crawl'])
self.assertEquals(re.findall(regex, 'stop:spider:app'),
['stop:spider:app'])
self.assertEquals(re.findall(regex, 'stop:stuff'), [])
def test_stop_monitor_mini_purge(self):
self.plugin.redis_conn.scan_iter = MagicMock(return_value=['link:istresearch.com:queue'])
self.plugin.redis_conn.zscan_iter = MagicMock(return_value=[
['{"crawlid":"crawl", "appid":"app"}'],
['{"crawlid":"crawl", "appid":"foo"}'],
])
self.assertEquals(self.plugin._mini_purge("link", "app", "crawl"), 1)
def setUp(self):
self.plugin = StopMonitor()
self.plugin.redis_conn = MagicMock()
self.plugin.logger = MagicMock()
def setUp(self):
self.plugin = StopMonitor()
self.plugin.redis_conn = MagicMock()
self.plugin.logger = MagicMock()
