def setUp(self):
self.plugin = InfoMonitor()
self.plugin.redis_conn = MagicMock()
self.plugin.logger = MagicMock()
class TestInfoPlugin(TestCase, RegexFixer):
def setUp(self):
self.plugin = InfoMonitor()
self.plugin.redis_conn = MagicMock()
self.plugin.logger = MagicMock()
def test_info_regex(self):
regex = self.fix_re(self.plugin.regex)
self.assertEquals(re.findall(regex, "info:stuff:stuff"), ["info:stuff:stuff"])
self.assertEquals(re.findall(regex, "info:stuff:stuff:stuff"), ["info:stuff:stuff:stuff"])
self.assertEquals(re.findall(regex, "info:stuff"), [])
def test_info_get_bin(self):
v1 = "stuff"
v1 = pickle.dumps(v1)
v2 = 200
self.plugin.redis_conn.zscan_iter = MagicMock(return_value=[(v1, v2)])
ret_val = self.plugin._get_bin("key")
self.assertEquals(ret_val, {-200: ["stuff"]})
def test_info_get_crawlid(self):
master = {}
master["uuid"] = "ABC123"
master["total_pending"] = 0
master["server_time"] = 5
master["crawlid"] = "crawlIDHERE"
elements = "info:link:testapp:crawlIDHERE".split(":")
dict = {}
dict["spiderid"] = elements[1]
dict["appid"] = elements[2]
dict["crawlid"] = elements[3]
self.plugin.redis_conn.exists = MagicMock(return_value=True)
self.plugin.redis_conn.get = MagicMock(return_value=10)
self.plugin.redis_conn.scan_iter = MagicMock(return_value=["theKey:bingo.com"])
self.plugin._get_bin = MagicMock(
return_value={-200: [{"appid": "testapp", "priority": 10, "crawlid": "crawlIDHERE"}]}
)
result = self.plugin._build_crawlid_info(master, dict)
success = {
"server_time": 5,
"crawlid": "crawlIDHERE",
"spiderid": "link",
"total_pending": 1,
"expires": 10,
"total_domains": 1,
"appid": "testapp",
"domains": {"bingo.com": {"low_priority": 10, "high_priority": 10, "total": 1}},
"uuid": "ABC123",
}
self.assertEquals(result, success)
def test_info_get_appid(self):
master = {}
master["uuid"] = "ABC123"
master["total_pending"] = 0
master["server_time"] = 5
elements = "info:link:testapp".split(":")
dict = {}
dict["spiderid"] = elements[1]
dict["appid"] = elements[2]
self.plugin.redis_conn.exists = MagicMock(return_value=True)
self.plugin.redis_conn.get = MagicMock(return_value=10)
self.plugin.redis_conn.scan_iter = MagicMock(return_value=["theKey:bingo.com"])
self.plugin._get_bin = MagicMock(return_value={-200: [{"appid": "testapp", "priority": 20, "crawlid": "cool"}]})
result = self.plugin._build_appid_info(master, dict)
success = {
"server_time": 5,
"uuid": "ABC123",
"total_pending": 1,
"total_domains": 1,
"total_crawlids": 1,
"appid": "testapp",
"spiderid": "link",
"crawlids": {
"cool": {
"domains": {"bingo.com": {"low_priority": 20, "high_priority": 20, "total": 1}},
"distinct_domains": 1,
"total": 1,
"expires": 10,
}
},
}
self.assertEquals(result, success)
class TestInfoPlugin(TestCase, RegexFixer):
def setUp(self):
self.plugin = InfoMonitor()
self.plugin.redis_conn = MagicMock()
self.plugin.logger = MagicMock()
def test_info_regex(self):
regex = self.fix_re(self.plugin.regex)
self.assertEquals(re.findall(regex, 'info:stuff:stuff'), ['info:stuff:stuff'])
self.assertEquals(re.findall(regex, 'info:stuff:stuff:stuff'), ['info:stuff:stuff:stuff'])
self.assertEquals(re.findall(regex, 'info:stuff'), [])
def test_info_get_bin(self):
v1 = "stuff"
v1 = pickle.dumps(v1)
v2 = 200
self.plugin.redis_conn.zscan_iter = MagicMock(return_value=[(v1, v2)])
ret_val = self.plugin._get_bin('key')
self.assertEquals(ret_val, {-200: ['stuff']})
def test_info_get_crawlid(self):
master = {}
master['uuid'] = 'ABC123'
master['total_pending'] = 0
master['server_time'] = 5
master['crawlid'] = "crawlIDHERE"
elements = 'info:link:testapp:crawlIDHERE'.split(":")
dict = {}
dict['spiderid'] = elements[1]
dict['appid'] = elements[2]
dict['crawlid'] = elements[3]
self.plugin.redis_conn.exists = MagicMock(return_value=True)
self.plugin.redis_conn.get = MagicMock(return_value=10)
self.plugin.redis_conn.scan_iter = MagicMock(return_value=['theKey:bingo.com'])
self.plugin._get_bin = MagicMock(return_value={-200: [{
'appid': "testapp", "priority": 10,
'crawlid': 'crawlIDHERE'}]})
result = self.plugin._build_crawlid_info(master, dict)
success = {
'server_time': 5,
'crawlid': 'crawlIDHERE',
'spiderid': 'link',
'total_pending': 1,
'expires': 10,
'total_domains': 1,
'appid': 'testapp',
'domains': {
'bingo.com': {
'low_priority': 10,
'high_priority': 10,
'total': 1
}},
'uuid': 'ABC123'
}
self.assertEquals(result, success)
def test_info_get_appid(self):
master = {}
master['uuid'] = 'ABC123'
master['total_pending'] = 0
master['server_time'] = 5
elements = 'info:link:testapp'.split(":")
dict = {}
dict['spiderid'] = elements[1]
dict['appid'] = elements[2]
self.plugin.redis_conn.exists = MagicMock(return_value=True)
self.plugin.redis_conn.get = MagicMock(return_value=10)
self.plugin.redis_conn.scan_iter = MagicMock(return_value=[
'theKey:bingo.com'])
self.plugin._get_bin = MagicMock(return_value={-200: [{
'appid': "testapp", "priority": 20,
'crawlid': 'cool'}]})
result = self.plugin._build_appid_info(master, dict)
success = {
'server_time': 5,
'uuid': 'ABC123',
'total_pending': 1,
'total_domains': 1,
'total_crawlids': 1,
'appid': 'testapp',
'spiderid': 'link',
'crawlids': {
'cool': {
'domains': {
'bingo.com': {
'low_priority': 20,
'high_priority': 20,
'total': 1
}},
'distinct_domains': 1,
'total': 1,
'expires': 10
}}}
self.assertEquals(result, success)
def setUp(self):
self.plugin = InfoMonitor()
self.plugin.redis_conn = MagicMock()
self.plugin.logger = MagicMock()
class TestInfoPlugin(TestCase, RegexFixer):
def setUp(self):
self.plugin = InfoMonitor()
self.plugin.redis_conn = MagicMock()
self.plugin.logger = MagicMock()
def test_info_regex(self):
regex = self.fix_re(self.plugin.regex)
self.assertEquals(re.findall(regex, 'info:stuff:stuff'), ['info:stuff:stuff'])
self.assertEquals(re.findall(regex, 'info:stuff:stuff:stuff'), ['info:stuff:stuff:stuff'])
self.assertEquals(re.findall(regex, 'info:stuff'), [])
def test_info_get_bin(self):
v1 = "stuff"
v1 = ujson.dumps(v1)
v2 = 200
self.plugin.redis_conn.zscan_iter = MagicMock(return_value=[(v1, v2)])
ret_val = self.plugin._get_bin('key')
self.assertEquals(ret_val, {-200: ['stuff']})
def test_info_get_crawlid(self):
master = {}
master['uuid'] = 'ABC123'
master['total_pending'] = 0
master['server_time'] = 5
master['crawlid'] = "crawlIDHERE"
elements = 'info:link:testapp:crawlIDHERE'.split(":")
dict = {}
dict['spiderid'] = elements[1]
dict['appid'] = elements[2]
dict['crawlid'] = elements[3]
self.plugin.redis_conn.exists = MagicMock(return_value=True)
self.plugin.redis_conn.get = MagicMock(return_value=10)
self.plugin.redis_conn.scan_iter = MagicMock(return_value=['theKey:bingo.com'])
self.plugin._get_bin = MagicMock(return_value={-200: [{
'appid': "testapp", "priority": 10,
'crawlid': 'crawlIDHERE'}]})
result = self.plugin._build_crawlid_info(master, dict)
success = {
'server_time': 5,
'crawlid': 'crawlIDHERE',
'spiderid': 'link',
'total_pending': 1,
'expires': 10,
'total_domains': 1,
'appid': 'testapp',
'domains': {
'bingo.com': {
'low_priority': 10,
'high_priority': 10,
'total': 1
}},
'uuid': 'ABC123'
}
self.assertEquals(result, success)
def test_info_get_appid(self):
master = {}
master['uuid'] = 'ABC123'
master['total_pending'] = 0
master['server_time'] = 5
elements = 'info:link:testapp'.split(":")
dict = {}
dict['spiderid'] = elements[1]
dict['appid'] = elements[2]
self.plugin.redis_conn.scan_iter = MagicMock(return_value=[
'theKey:bingo.com'])
self.plugin._get_bin = MagicMock(return_value={-200: [{
'appid': "testapp", "priority": 20,
'crawlid': 'cool', 'expires': 10}]})
result = self.plugin._build_appid_info(master, dict)
success = {
'server_time': 5,
'uuid': 'ABC123',
'total_pending': 1,
'total_domains': 1,
'total_crawlids': 1,
'appid': 'testapp',
'spiderid': 'link',
'crawlids': {
'cool': {
'domains': {
'bingo.com': {
'low_priority': 20,
'high_priority': 20,
'total': 1
}},
'distinct_domains': 1,
'total': 1,
'expires': 10
}}}
self.assertEquals(result, success)