Esempio n. 1
0
 def setUp(self):
     self.plugin = InfoMonitor()
     self.plugin.redis_conn = MagicMock()
     self.plugin.logger = MagicMock()
Esempio n. 2
0
class TestInfoPlugin(TestCase, RegexFixer):
    def setUp(self):
        self.plugin = InfoMonitor()
        self.plugin.redis_conn = MagicMock()
        self.plugin.logger = MagicMock()

    def test_info_regex(self):
        regex = self.fix_re(self.plugin.regex)
        self.assertEquals(re.findall(regex, "info:stuff:stuff"), ["info:stuff:stuff"])
        self.assertEquals(re.findall(regex, "info:stuff:stuff:stuff"), ["info:stuff:stuff:stuff"])
        self.assertEquals(re.findall(regex, "info:stuff"), [])

    def test_info_get_bin(self):
        v1 = "stuff"
        v1 = pickle.dumps(v1)
        v2 = 200
        self.plugin.redis_conn.zscan_iter = MagicMock(return_value=[(v1, v2)])
        ret_val = self.plugin._get_bin("key")
        self.assertEquals(ret_val, {-200: ["stuff"]})

    def test_info_get_crawlid(self):
        master = {}
        master["uuid"] = "ABC123"
        master["total_pending"] = 0
        master["server_time"] = 5
        master["crawlid"] = "crawlIDHERE"

        elements = "info:link:testapp:crawlIDHERE".split(":")
        dict = {}
        dict["spiderid"] = elements[1]
        dict["appid"] = elements[2]
        dict["crawlid"] = elements[3]

        self.plugin.redis_conn.exists = MagicMock(return_value=True)
        self.plugin.redis_conn.get = MagicMock(return_value=10)
        self.plugin.redis_conn.scan_iter = MagicMock(return_value=["theKey:bingo.com"])
        self.plugin._get_bin = MagicMock(
            return_value={-200: [{"appid": "testapp", "priority": 10, "crawlid": "crawlIDHERE"}]}
        )

        result = self.plugin._build_crawlid_info(master, dict)

        success = {
            "server_time": 5,
            "crawlid": "crawlIDHERE",
            "spiderid": "link",
            "total_pending": 1,
            "expires": 10,
            "total_domains": 1,
            "appid": "testapp",
            "domains": {"bingo.com": {"low_priority": 10, "high_priority": 10, "total": 1}},
            "uuid": "ABC123",
        }

        self.assertEquals(result, success)

    def test_info_get_appid(self):
        master = {}
        master["uuid"] = "ABC123"
        master["total_pending"] = 0
        master["server_time"] = 5
        elements = "info:link:testapp".split(":")
        dict = {}
        dict["spiderid"] = elements[1]
        dict["appid"] = elements[2]

        self.plugin.redis_conn.exists = MagicMock(return_value=True)
        self.plugin.redis_conn.get = MagicMock(return_value=10)
        self.plugin.redis_conn.scan_iter = MagicMock(return_value=["theKey:bingo.com"])
        self.plugin._get_bin = MagicMock(return_value={-200: [{"appid": "testapp", "priority": 20, "crawlid": "cool"}]})

        result = self.plugin._build_appid_info(master, dict)

        success = {
            "server_time": 5,
            "uuid": "ABC123",
            "total_pending": 1,
            "total_domains": 1,
            "total_crawlids": 1,
            "appid": "testapp",
            "spiderid": "link",
            "crawlids": {
                "cool": {
                    "domains": {"bingo.com": {"low_priority": 20, "high_priority": 20, "total": 1}},
                    "distinct_domains": 1,
                    "total": 1,
                    "expires": 10,
                }
            },
        }

        self.assertEquals(result, success)
Esempio n. 3
0
class TestInfoPlugin(TestCase, RegexFixer):

    def setUp(self):
        self.plugin = InfoMonitor()
        self.plugin.redis_conn = MagicMock()
        self.plugin.logger = MagicMock()

    def test_info_regex(self):
        regex = self.fix_re(self.plugin.regex)
        self.assertEquals(re.findall(regex, 'info:stuff:stuff'), ['info:stuff:stuff'])
        self.assertEquals(re.findall(regex, 'info:stuff:stuff:stuff'), ['info:stuff:stuff:stuff'])
        self.assertEquals(re.findall(regex, 'info:stuff'), [])

    def test_info_get_bin(self):
        v1 = "stuff"
        v1 = pickle.dumps(v1)
        v2 = 200
        self.plugin.redis_conn.zscan_iter = MagicMock(return_value=[(v1, v2)])
        ret_val = self.plugin._get_bin('key')
        self.assertEquals(ret_val, {-200: ['stuff']})

    def test_info_get_crawlid(self):
        master = {}
        master['uuid'] = 'ABC123'
        master['total_pending'] = 0
        master['server_time'] = 5
        master['crawlid'] = "crawlIDHERE"

        elements = 'info:link:testapp:crawlIDHERE'.split(":")
        dict = {}
        dict['spiderid'] = elements[1]
        dict['appid'] = elements[2]
        dict['crawlid'] = elements[3]

        self.plugin.redis_conn.exists = MagicMock(return_value=True)
        self.plugin.redis_conn.get = MagicMock(return_value=10)
        self.plugin.redis_conn.scan_iter = MagicMock(return_value=['theKey:bingo.com'])
        self.plugin._get_bin = MagicMock(return_value={-200: [{
                                         'appid': "testapp", "priority": 10,
                                         'crawlid': 'crawlIDHERE'}]})

        result = self.plugin._build_crawlid_info(master, dict)

        success = {
            'server_time': 5,
            'crawlid': 'crawlIDHERE',
            'spiderid': 'link',
            'total_pending': 1,
            'expires': 10,
            'total_domains': 1,
            'appid': 'testapp',
            'domains': {
                'bingo.com': {
                    'low_priority': 10,
                    'high_priority': 10,
                    'total': 1
                }},
            'uuid': 'ABC123'
        }

        self.assertEquals(result, success)

    def test_info_get_appid(self):
        master = {}
        master['uuid'] = 'ABC123'
        master['total_pending'] = 0
        master['server_time'] = 5
        elements = 'info:link:testapp'.split(":")
        dict = {}
        dict['spiderid'] = elements[1]
        dict['appid'] = elements[2]

        self.plugin.redis_conn.exists = MagicMock(return_value=True)
        self.plugin.redis_conn.get = MagicMock(return_value=10)
        self.plugin.redis_conn.scan_iter = MagicMock(return_value=[
                                                     'theKey:bingo.com'])
        self.plugin._get_bin = MagicMock(return_value={-200: [{
                                         'appid': "testapp", "priority": 20,
                                         'crawlid': 'cool'}]})

        result = self.plugin._build_appid_info(master, dict)

        success = {
            'server_time': 5,
            'uuid': 'ABC123',
            'total_pending': 1,
            'total_domains': 1,
            'total_crawlids': 1,
            'appid': 'testapp',
            'spiderid': 'link',
            'crawlids': {
                'cool': {
                    'domains': {
                        'bingo.com': {
                            'low_priority': 20,
                            'high_priority': 20,
                            'total': 1
                    }},
                    'distinct_domains': 1,
                    'total': 1,
                    'expires': 10
                }}}

        self.assertEquals(result, success)
Esempio n. 4
0
 def setUp(self):
     self.plugin = InfoMonitor()
     self.plugin.redis_conn = MagicMock()
     self.plugin.logger = MagicMock()
Esempio n. 5
0
class TestInfoPlugin(TestCase, RegexFixer):

    def setUp(self):
        self.plugin = InfoMonitor()
        self.plugin.redis_conn = MagicMock()
        self.plugin.logger = MagicMock()

    def test_info_regex(self):
        regex = self.fix_re(self.plugin.regex)
        self.assertEquals(re.findall(regex, 'info:stuff:stuff'), ['info:stuff:stuff'])
        self.assertEquals(re.findall(regex, 'info:stuff:stuff:stuff'), ['info:stuff:stuff:stuff'])
        self.assertEquals(re.findall(regex, 'info:stuff'), [])

    def test_info_get_bin(self):
        v1 = "stuff"
        v1 = ujson.dumps(v1)
        v2 = 200
        self.plugin.redis_conn.zscan_iter = MagicMock(return_value=[(v1, v2)])
        ret_val = self.plugin._get_bin('key')
        self.assertEquals(ret_val, {-200: ['stuff']})

    def test_info_get_crawlid(self):
        master = {}
        master['uuid'] = 'ABC123'
        master['total_pending'] = 0
        master['server_time'] = 5
        master['crawlid'] = "crawlIDHERE"

        elements = 'info:link:testapp:crawlIDHERE'.split(":")
        dict = {}
        dict['spiderid'] = elements[1]
        dict['appid'] = elements[2]
        dict['crawlid'] = elements[3]

        self.plugin.redis_conn.exists = MagicMock(return_value=True)
        self.plugin.redis_conn.get = MagicMock(return_value=10)
        self.plugin.redis_conn.scan_iter = MagicMock(return_value=['theKey:bingo.com'])
        self.plugin._get_bin = MagicMock(return_value={-200: [{
                                         'appid': "testapp", "priority": 10,
                                         'crawlid': 'crawlIDHERE'}]})

        result = self.plugin._build_crawlid_info(master, dict)

        success = {
            'server_time': 5,
            'crawlid': 'crawlIDHERE',
            'spiderid': 'link',
            'total_pending': 1,
            'expires': 10,
            'total_domains': 1,
            'appid': 'testapp',
            'domains': {
                'bingo.com': {
                    'low_priority': 10,
                    'high_priority': 10,
                    'total': 1
                }},
            'uuid': 'ABC123'
        }

        self.assertEquals(result, success)

    def test_info_get_appid(self):
        master = {}
        master['uuid'] = 'ABC123'
        master['total_pending'] = 0
        master['server_time'] = 5
        elements = 'info:link:testapp'.split(":")
        dict = {}
        dict['spiderid'] = elements[1]
        dict['appid'] = elements[2]

        self.plugin.redis_conn.scan_iter = MagicMock(return_value=[
                                                     'theKey:bingo.com'])
        self.plugin._get_bin = MagicMock(return_value={-200: [{
                                         'appid': "testapp", "priority": 20,
                                         'crawlid': 'cool', 'expires': 10}]})

        result = self.plugin._build_appid_info(master, dict)

        success = {
            'server_time': 5,
            'uuid': 'ABC123',
            'total_pending': 1,
            'total_domains': 1,
            'total_crawlids': 1,
            'appid': 'testapp',
            'spiderid': 'link',
            'crawlids': {
                'cool': {
                    'domains': {
                        'bingo.com': {
                            'low_priority': 20,
                            'high_priority': 20,
                            'total': 1
                    }},
                    'distinct_domains': 1,
                    'total': 1,
                    'expires': 10
                }}}

        self.assertEquals(result, success)