def test_stop(self): """ Test .stop() """ sel = Selector(gt10, is_even, self.gens) sel.stop() self.assertEqual([], sel.gens) self.assertIsNone(sel.started) self.assertIsNone(sel.curr)
def test_stop(self): """ Test .stop() """ sel = Selector(gt10, is_even, self.gens) sel.stop() self.assertEqual([], sel.gens) self.assertIsNone(sel.started) self.assertIsNone(sel.curr)
class Scheduler(object): def __init__(self, conf): self.conf = conf self.log = conf.get('log') self.site_statistic = SiteStatistics(conf['site_task_collect_db']) self.seed_statistic = SeedStatistics(conf['seed_task_collect_db']) self.scheduler = SiteSchedulerManager(self.conf, self.site_statistic, self.seed_statistic) self.selector = Selector(self.scheduler, self.conf) self.crawl_selector = CrawlSelector(self.log, conf['selector_conf'], conf['beanstalk_conf'], self.scheduler) def schedule_task(self, task): if task: self.scheduler.schedule_task(task) def select_seed(self): try: self.scheduler.select_seed() except Exception: self.log.error('start_on_site_tasks\terror:%s' % str(traceback.format_exc())) return False def start_one_site_tasks(self, site): try: self.log.info('start_one_site_tasks\tsite:%s starting' % site) return self.scheduler.start_one_site_tasks(site) except Exception: self.log.error('start_on_site_tasks\tsite:%s\terror:%s' % (site, str(traceback.format_exc()))) return False def stop_one_site_tasks(self, site): self.log.info('site:%s stopping...' % site) return self.scheduler.stop_one_site_tasks(site) def clear_one_site_cache(self, site): self.log.info('site:%s clear cache...' % site) return self.scheduler.clear_one_site_cache(site) def restart_seed(self, seed_id, site): self.log.info('seed:%s site:%s restart...' % (seed_id, site)) return self.scheduler.restart_seed(seed_id, site) def start(self): self.scheduler.start() self.crawl_selector.start() self.selector.start() self.site_statistic.start() self.seed_statistic.start() self.log.info('scheduler start successful') def save_status(self): self.scheduler.save_status() self.selector.stop() self.crawl_selector.stop() self.site_statistic.stop() self.seed_statistic.stop() def stop(self): self.scheduler.stop() self.selector.stop() self.crawl_selector.stop() self.site_statistic.stop() self.seed_statistic.stop()