class QueuePollerTest(unittest.TestCase):
def setUp(self):
d = self.mktemp()
eggs_dir = os.path.join(d, 'eggs')
dbs_dir = os.path.join(d, 'dbs')
os.makedirs(eggs_dir)
os.makedirs(dbs_dir)
os.makedirs(os.path.join(eggs_dir, 'mybot1'))
os.makedirs(os.path.join(eggs_dir, 'mybot2'))
config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
self.queues = get_spider_queues(config)
self.poller = QueuePoller(config)
self.lancher_mock = Mock()
self.lancher_mock.processes = {}
def test_interface(self):
verifyObject(IPoller, self.poller)
def test_poll_next(self):
self.queues['mybot1'].add('spider1')
self.queues['mybot2'].add('spider2')
d1 = self.poller.next()
d2 = self.poller.next()
self.failUnless(isinstance(d1, Deferred))
self.failIf(hasattr(d1, 'result'))
self.poller.poll(self.lancher_mock)
self.queues['mybot1'].pop()
self.poller.poll(self.lancher_mock)
self.failUnlessEqual(d1.result, {'_project': 'mybot1', '_spider': 'spider1'})
self.failUnlessEqual(d2.result, {'_project': 'mybot2', '_spider': 'spider2'})
class QueuePollerTest(unittest.TestCase):
def setUp(self):
d = self.mktemp()
eggs_dir = os.path.join(d, "eggs")
dbs_dir = os.path.join(d, "dbs")
os.makedirs(eggs_dir)
os.makedirs(dbs_dir)
os.makedirs(os.path.join(eggs_dir, "mybot1"))
os.makedirs(os.path.join(eggs_dir, "mybot2"))
config = Config(values={"eggs_dir": eggs_dir, "dbs_dir": dbs_dir})
self.queues = get_spider_queues(config)
self.poller = QueuePoller(config)
def test_interface(self):
verifyObject(IPoller, self.poller)
def test_poll_next(self):
self.queues["mybot1"].add("spider1")
self.queues["mybot2"].add("spider2")
d1 = self.poller.next()
d2 = self.poller.next()
self.failUnless(isinstance(d1, Deferred))
self.failIf(hasattr(d1, "result"))
self.poller.poll()
self.queues["mybot1"].pop()
self.poller.poll()
self.failUnlessEqual(d1.result, {"project": "mybot1"})
self.failUnlessEqual(d2.result, {"project": "mybot2"})
class QueuePollerTest(unittest.TestCase):
def setUp(self):
d = self.mktemp()
eggs_dir = os.path.join(d, 'eggs')
dbs_dir = os.path.join(d, 'dbs')
os.makedirs(eggs_dir)
os.makedirs(dbs_dir)
os.makedirs(os.path.join(eggs_dir, 'mybot1'))
os.makedirs(os.path.join(eggs_dir, 'mybot2'))
config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
self.queues = get_spider_queues(config)
self.poller = QueuePoller(config)
def test_interface(self):
verifyObject(IPoller, self.poller)
def test_poll_next(self):
self.queues['mybot1'].add('spider1')
self.queues['mybot2'].add('spider2')
d1 = self.poller.next()
d2 = self.poller.next()
self.failUnless(isinstance(d1, Deferred))
self.failIf(hasattr(d1, 'result'))
self.poller.poll()
self.queues['mybot1'].pop()
self.poller.poll()
self.failUnlessEqual(d1.result, {'_project': 'mybot1', '_spider': 'spider1'})
self.failUnlessEqual(d2.result, {'_project': 'mybot2', '_spider': 'spider2'})
def setUp(self):
d = self.mktemp()
eggs_dir = os.path.join(d, 'eggs')
dbs_dir = os.path.join(d, 'dbs')
os.makedirs(eggs_dir)
os.makedirs(dbs_dir)
os.makedirs(os.path.join(eggs_dir, 'mybot1'))
os.makedirs(os.path.join(eggs_dir, 'mybot2'))
config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
self.queues = get_spider_queues(config)
self.poller = QueuePoller(config)
class QueuePollerTest(unittest.TestCase):
def setUp(self):
d = self.mktemp()
eggs_dir = os.path.join(d, 'eggs')
dbs_dir = os.path.join(d, 'dbs')
os.makedirs(eggs_dir)
os.makedirs(dbs_dir)
os.makedirs(os.path.join(eggs_dir, 'mybot1'))
os.makedirs(os.path.join(eggs_dir, 'mybot2'))
config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
self.queues = get_spider_queues(config)
self.poller = QueuePoller(config)
def test_interface(self):
verifyObject(IPoller, self.poller)
def test_poll_next(self):
cfg = {'mybot1': 'spider1',
'mybot2': 'spider2'}
priority = 0
for prj, spd in cfg.items():
self.queues[prj].add(spd, priority)
d1 = self.poller.next()
d2 = self.poller.next()
self.assertIsInstance(d1, Deferred)
self.assertFalse(hasattr(d1, 'result'))
# poll once
self.poller.poll()
self.assertTrue(hasattr(d1, 'result'))
self.assertTrue(getattr(d1, 'called', False))
# which project got run: project1 or project2?
self.assertTrue(d1.result.get('_project'))
prj = d1.result['_project']
self.assertEqual(d1.result['_spider'], cfg.pop(prj))
self.queues[prj].pop()
# poll twice
# check that the other project's spider got to run
self.poller.poll()
prj, spd = cfg.popitem()
self.assertEqual(d2.result, {'_project': prj, '_spider': spd})
class QueuePollerTest(unittest.TestCase):
def setUp(self):
d = self.mktemp()
eggs_dir = os.path.join(d, 'eggs')
dbs_dir = os.path.join(d, 'dbs')
os.makedirs(eggs_dir)
os.makedirs(dbs_dir)
os.makedirs(os.path.join(eggs_dir, 'mybot1'))
os.makedirs(os.path.join(eggs_dir, 'mybot2'))
config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
self.queues = get_spider_queues(config)
self.poller = QueuePoller(config)
def test_interface(self):
verifyObject(IPoller, self.poller)
def test_poll_next(self):
cfg = {'mybot1': 'spider1',
'mybot2': 'spider2'}
for prj, spd in cfg.items():
self.queues[prj].add(spd)
d1 = self.poller.next()
d2 = self.poller.next()
self.failUnless(isinstance(d1, Deferred))
self.failIf(hasattr(d1, 'result'))
# poll once
self.poller.poll()
self.failUnless(hasattr(d1, 'result') and getattr(d1, 'called', False))
# which project got run: project1 or project2?
self.failUnless(d1.result.get('_project'))
prj = d1.result['_project']
self.failUnlessEqual(d1.result['_spider'], cfg.pop(prj))
self.queues[prj].pop()
# poll twice
# check that the other project's spider got to run
self.poller.poll()
prj, spd = cfg.popitem()
self.failUnlessEqual(d2.result, {'_project': prj, '_spider': spd})
def setUp(self):
d = self.mktemp()
eggs_dir = os.path.join(d, "eggs")
dbs_dir = os.path.join(d, "dbs")
os.makedirs(eggs_dir)
os.makedirs(dbs_dir)
os.makedirs(os.path.join(eggs_dir, "mybot1"))
os.makedirs(os.path.join(eggs_dir, "mybot2"))
config = Config(values={"eggs_dir": eggs_dir, "dbs_dir": dbs_dir})
self.queues = get_spider_queues(config)
self.poller = QueuePoller(config)
def setUp(self):
d = self.mktemp()
eggs_dir = os.path.join(d, 'eggs')
dbs_dir = os.path.join(d, 'dbs')
os.makedirs(eggs_dir)
os.makedirs(dbs_dir)
os.makedirs(os.path.join(eggs_dir, 'mybot1'))
os.makedirs(os.path.join(eggs_dir, 'mybot2'))
config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
self.queues = get_spider_queues(config)
self.poller = QueuePoller(config)
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
jspath = config.get('jobstorage', 'scrapyd.jobstorage.MemoryJobStorage')
jscls = load_object(jspath)
jobstorage = jscls(config)
app.setComponent(IJobStorage, jobstorage)
eggstorage = config.get('eggstorage',
'scrapyd.eggstorage.FilesystemEggStorage')
eggstoragecls = load_object(eggstorage)
app.setComponent(IEggStorage, eggstoragecls(config))
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webpath = config.get('webroot', 'scrapyd.website.Root')
webcls = load_object(webpath)
resource = create_wrapped_resource(webcls, config, app)
webservice = TCPServer(http_port,
server.Site(resource),
interface=bind_address)
log.msg(
format=
"Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
schedpath = config.get('scheduler', 'scrapyd.scheduler.SpiderScheduler')
schedCls = load_object(schedpath)
scheduler = schedCls(config, app)
environment = Environment(config)
pubsub_path = config.get('pubsub', 'scrapyd.pubsub.BasePubSub')
pubsubCls = load_object(pubsub_path)
pubsub = pubsubCls(config, app)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
app.setComponent(IPubSub, pubsub)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port,
server.Site(Root(config, app)),
interface=bind_address)
log.msg(
format=
"Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
pubsub.setServiceParent(app)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
if 'PORT' in os.environ:
http_port = int(os.environ.get('PORT'))
bind_address = '0.0.0.0' if 'PORT' in os.environ else config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
webpath = config.get('webroot', 'scrapyd.website.Root')
webcls = load_object(webpath)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port, server.Site(webcls(config, app)), interface=bind_address)
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
