예제 #1
0
class QueuePollerTest(unittest.TestCase):
    def setUp(self):
        d = self.mktemp()
        eggs_dir = os.path.join(d, "eggs")
        dbs_dir = os.path.join(d, "dbs")
        os.makedirs(eggs_dir)
        os.makedirs(dbs_dir)
        os.makedirs(os.path.join(eggs_dir, "mybot1"))
        os.makedirs(os.path.join(eggs_dir, "mybot2"))
        config = Config(values={"eggs_dir": eggs_dir, "dbs_dir": dbs_dir})
        self.queues = get_spider_queues(config)
        self.poller = QueuePoller(config)

    def test_interface(self):
        verifyObject(IPoller, self.poller)

    def test_poll_next(self):
        self.queues["mybot1"].add("spider1")
        self.queues["mybot2"].add("spider2")
        d1 = self.poller.next()
        d2 = self.poller.next()
        self.failUnless(isinstance(d1, Deferred))
        self.failIf(hasattr(d1, "result"))
        self.poller.poll()
        self.queues["mybot1"].pop()
        self.poller.poll()
        self.failUnlessEqual(d1.result, {"project": "mybot1"})
        self.failUnlessEqual(d2.result, {"project": "mybot2"})
예제 #2
0
class QueuePollerTest(unittest.TestCase):

    def setUp(self):
        d = self.mktemp()
        eggs_dir = os.path.join(d, 'eggs')
        dbs_dir = os.path.join(d, 'dbs')
        os.makedirs(eggs_dir)
        os.makedirs(dbs_dir)
        os.makedirs(os.path.join(eggs_dir, 'mybot1'))
        os.makedirs(os.path.join(eggs_dir, 'mybot2'))
        config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
        self.queues = get_spider_queues(config)
        self.poller = QueuePoller(config)
        self.lancher_mock = Mock()
        self.lancher_mock.processes =  {}
                
    def test_interface(self):
        verifyObject(IPoller, self.poller)

    def test_poll_next(self):
        self.queues['mybot1'].add('spider1')
        self.queues['mybot2'].add('spider2')
        d1 = self.poller.next()
        d2 = self.poller.next()
        self.failUnless(isinstance(d1, Deferred))
        self.failIf(hasattr(d1, 'result'))
        self.poller.poll(self.lancher_mock)
        self.queues['mybot1'].pop()
        self.poller.poll(self.lancher_mock)
        self.failUnlessEqual(d1.result, {'_project': 'mybot1', '_spider': 'spider1'})
        self.failUnlessEqual(d2.result, {'_project': 'mybot2', '_spider': 'spider2'})
예제 #3
0
class QueuePollerTest(unittest.TestCase):

    def setUp(self):
        d = self.mktemp()
        eggs_dir = os.path.join(d, 'eggs')
        dbs_dir = os.path.join(d, 'dbs')
        os.makedirs(eggs_dir)
        os.makedirs(dbs_dir)
        os.makedirs(os.path.join(eggs_dir, 'mybot1'))
        os.makedirs(os.path.join(eggs_dir, 'mybot2'))
        config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
        self.queues = get_spider_queues(config)
        self.poller = QueuePoller(config)

    def test_interface(self):
        verifyObject(IPoller, self.poller)

    def test_poll_next(self):
        self.queues['mybot1'].add('spider1')
        self.queues['mybot2'].add('spider2')
        d1 = self.poller.next()
        d2 = self.poller.next()
        self.failUnless(isinstance(d1, Deferred))
        self.failIf(hasattr(d1, 'result'))
        self.poller.poll()
        self.queues['mybot1'].pop()
        self.poller.poll()
        self.failUnlessEqual(d1.result, {'_project': 'mybot1', '_spider': 'spider1'})
        self.failUnlessEqual(d2.result, {'_project': 'mybot2', '_spider': 'spider2'})
예제 #4
0
class QueuePollerTest(unittest.TestCase):

    def setUp(self):
        d = self.mktemp()
        eggs_dir = os.path.join(d, 'eggs')
        dbs_dir = os.path.join(d, 'dbs')
        os.makedirs(eggs_dir)
        os.makedirs(dbs_dir)
        os.makedirs(os.path.join(eggs_dir, 'mybot1'))
        os.makedirs(os.path.join(eggs_dir, 'mybot2'))
        config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
        self.queues = get_spider_queues(config)
        self.poller = QueuePoller(config)

    def test_interface(self):
        verifyObject(IPoller, self.poller)

    def test_poll_next(self):
        cfg = {'mybot1': 'spider1',
               'mybot2': 'spider2'}
        priority = 0
        for prj, spd in cfg.items():
            self.queues[prj].add(spd, priority)

        d1 = self.poller.next()
        d2 = self.poller.next()
        self.assertIsInstance(d1, Deferred)
        self.assertFalse(hasattr(d1, 'result'))

        # poll once
        self.poller.poll()
        self.assertTrue(hasattr(d1, 'result'))
        self.assertTrue(getattr(d1, 'called', False))

        # which project got run: project1 or project2?
        self.assertTrue(d1.result.get('_project'))
        prj = d1.result['_project']
        self.assertEqual(d1.result['_spider'], cfg.pop(prj))

        self.queues[prj].pop()

        # poll twice
        # check that the other project's spider got to run
        self.poller.poll()
        prj, spd = cfg.popitem()
        self.assertEqual(d2.result, {'_project': prj, '_spider': spd})
예제 #5
0
class QueuePollerTest(unittest.TestCase):

    def setUp(self):
        d = self.mktemp()
        eggs_dir = os.path.join(d, 'eggs')
        dbs_dir = os.path.join(d, 'dbs')
        os.makedirs(eggs_dir)
        os.makedirs(dbs_dir)
        os.makedirs(os.path.join(eggs_dir, 'mybot1'))
        os.makedirs(os.path.join(eggs_dir, 'mybot2'))
        config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
        self.queues = get_spider_queues(config)
        self.poller = QueuePoller(config)

    def test_interface(self):
        verifyObject(IPoller, self.poller)

    def test_poll_next(self):
        cfg = {'mybot1': 'spider1',
               'mybot2': 'spider2'}
        for prj, spd in cfg.items():
            self.queues[prj].add(spd)

        d1 = self.poller.next()
        d2 = self.poller.next()
        self.failUnless(isinstance(d1, Deferred))
        self.failIf(hasattr(d1, 'result'))

        # poll once
        self.poller.poll()
        self.failUnless(hasattr(d1, 'result') and getattr(d1, 'called', False))

        # which project got run: project1 or project2?
        self.failUnless(d1.result.get('_project'))
        prj = d1.result['_project']
        self.failUnlessEqual(d1.result['_spider'], cfg.pop(prj))

        self.queues[prj].pop()

        # poll twice
        # check that the other project's spider got to run
        self.poller.poll()
        prj, spd = cfg.popitem()
        self.failUnlessEqual(d2.result, {'_project': prj, '_spider': spd})