Python get_spider_queues Examples

Example #1

File: test_poller.py Project: pkufranky/scrapy

 def setUp(self):
     d = self.mktemp()
     eggs_dir = os.path.join(d, "eggs")
     dbs_dir = os.path.join(d, "dbs")
     os.makedirs(eggs_dir)
     os.makedirs(dbs_dir)
     os.makedirs(os.path.join(eggs_dir, "mybot1"))
     os.makedirs(os.path.join(eggs_dir, "mybot2"))
     config = Config(values={"eggs_dir": eggs_dir, "dbs_dir": dbs_dir})
     self.queues = get_spider_queues(config)
     self.poller = QueuePoller(config)

Example #2

File: launcher.py Project: xutaoding/csf_scrapyd

    def __init__(self, config, app):
        self.processes = {}
        self.finished = []
        self.finished_to_keep = config.getint('finished_to_keep', 100)
        self.max_proc = self._get_max_proc(config)
        self.runner = config.get('runner', 'scrapyd.runner')
        self.app = app
        self.scheduler_persist = get_scheduler_persist(config)
        self.spider_queue = get_spider_queues(config)

        self.scheduler = self.app.getComponent(ISpiderScheduler)

Example #3

File: test_poller.py Project: ToGoBananas/scrapyd

 def setUp(self):
     d = self.mktemp()
     eggs_dir = os.path.join(d, 'eggs')
     dbs_dir = os.path.join(d, 'dbs')
     os.makedirs(eggs_dir)
     os.makedirs(dbs_dir)
     os.makedirs(os.path.join(eggs_dir, 'mybot1'))
     os.makedirs(os.path.join(eggs_dir, 'mybot2'))
     config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
     self.queues = get_spider_queues(config)
     self.poller = QueuePoller(config)

Example #4

File: test_poller.py Project: 535521469/crawler_sth

 def setUp(self):
     d = self.mktemp()
     eggs_dir = os.path.join(d, 'eggs')
     dbs_dir = os.path.join(d, 'dbs')
     os.makedirs(eggs_dir)
     os.makedirs(dbs_dir)
     os.makedirs(os.path.join(eggs_dir, 'mybot1'))
     os.makedirs(os.path.join(eggs_dir, 'mybot2'))
     config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
     self.queues = get_spider_queues(config)
     self.poller = QueuePoller(config)

Example #5

File: scheduler.py Project: drankinn/scrapyd

 def update_projects(self):
     self.queues = get_spider_queues(self.config)

Example #6

 def update_projects(self):
     self.queues = get_spider_queues(self.config)