def test_call_later_once(self): self.called = 0 reactor = Clock() call = CallLaterOnce(self.call_function, reactor=reactor) call.schedule(delay=1) reactor.advance(1) assert self.called == 1
def test_call_later_twice_already_scheduled(self): self.called = 0 reactor = Clock() call = CallLaterOnce(self.call_function, reactor=reactor) call.schedule(delay=1) call.schedule(delay=2) reactor.advance(2) assert self.called == 1
def __init__(self, new_batch, consume_incoming, consume_scoring, no_batches, no_scoring_log, new_batch_delay, no_spider_log): self.new_batch = CallLaterOnce(new_batch) self.new_batch.setErrback(self.error) self.consumption = CallLaterOnce(consume_incoming) self.consumption.setErrback(self.error) self.scheduling = CallLaterOnce(self.schedule) self.scheduling.setErrback(self.error) self.scoring_consumption = CallLaterOnce(consume_scoring) self.scoring_consumption.setErrback(self.error) self.no_batches = no_batches self.no_scoring_log = no_scoring_log self.no_spider_log = no_spider_log self.new_batch_delay = new_batch_delay
def __init__(self, new_batch, consume_incoming, consume_scoring, no_batches, enable_strategy_worker, new_batch_delay, no_incoming): self.new_batch = CallLaterOnce(new_batch) self.new_batch.setErrback(self.error) self.consumption = CallLaterOnce(consume_incoming) self.consumption.setErrback(self.error) self.scheduling = CallLaterOnce(self.schedule) self.scheduling.setErrback(self.error) self.scoring_consumption = CallLaterOnce(consume_scoring) self.scoring_consumption.setErrback(self.error) self.disable_new_batches = no_batches self.disable_scoring_consumption = not enable_strategy_worker self.disable_incoming = no_incoming self.new_batch_delay = new_batch_delay
class DBWorkerPeriodicComponent(DBWorkerBaseComponent): def __init__(self, worker, settings, stop_event, *args, **kwargs): super(DBWorkerPeriodicComponent, self).__init__(worker, settings, stop_event) self.periodic_task = CallLaterOnce(self.run_and_reschedule) self.periodic_task.setErrback(self.run_errback) def schedule(self, delay=0): self.periodic_task.schedule(delay) def run_and_reschedule(self): if not self.stopped: self.run() self.periodic_task.schedule() def run_errback(self, failure): self.logger.error(failure.getTraceback()) if not self.stopped: self.periodic_task.schedule() @property def stopped(self): return self.stop_event.is_set()
def __init__(self, worker, settings, stop_event, *args, **kwargs): super(DBWorkerPeriodicComponent, self).__init__(worker, settings, stop_event) self.periodic_task = CallLaterOnce(self.run_and_reschedule) self.periodic_task.setErrback(self.run_errback)
class Slot(object): def __init__(self, new_batch, consume_incoming, consume_scoring, no_batches, enable_strategy_worker, new_batch_delay, no_incoming): self.new_batch = CallLaterOnce(new_batch) self.new_batch.setErrback(self.error) self.consumption = CallLaterOnce(consume_incoming) self.consumption.setErrback(self.error) self.scheduling = CallLaterOnce(self.schedule) self.scheduling.setErrback(self.error) self.scoring_consumption = CallLaterOnce(consume_scoring) self.scoring_consumption.setErrback(self.error) self.disable_new_batches = no_batches self.disable_scoring_consumption = not enable_strategy_worker self.disable_incoming = no_incoming self.new_batch_delay = new_batch_delay def error(self, f): logger.exception(f.value) return f def schedule(self, on_start=False): if on_start and not self.disable_new_batches: self.new_batch.schedule(0) if not self.disable_incoming: self.consumption.schedule() if not self.disable_new_batches: self.new_batch.schedule(self.new_batch_delay) if not self.disable_scoring_consumption: self.scoring_consumption.schedule() self.scheduling.schedule(5.0)
class Slot(object): def __init__(self, new_batch, consume_incoming, consume_scoring, no_batches, no_scoring_log, new_batch_delay, no_spider_log): self.new_batch = CallLaterOnce(new_batch) self.new_batch.setErrback(self.error) self.consumption = CallLaterOnce(consume_incoming) self.consumption.setErrback(self.error) self.scheduling = CallLaterOnce(self.schedule) self.scheduling.setErrback(self.error) self.scoring_consumption = CallLaterOnce(consume_scoring) self.scoring_consumption.setErrback(self.error) self.no_batches = no_batches self.no_scoring_log = no_scoring_log self.no_spider_log = no_spider_log self.new_batch_delay = new_batch_delay def error(self, f): logger.exception(f.value) return f def schedule(self, on_start=False): if on_start and not self.no_batches: self.new_batch.schedule(0) if not self.no_spider_log: self.consumption.schedule() if not self.no_batches: self.new_batch.schedule(self.new_batch_delay) if not self.no_scoring_log: self.scoring_consumption.schedule() self.scheduling.schedule(5.0) def cancel(self): self.scheduling.cancel() self.scoring_consumption.cancel() self.new_batch.cancel() self.consumption.cancel()
class Slot(object): def __init__(self, new_batch, consume_incoming, consume_scoring, no_batches, no_scoring_log, new_batch_delay, no_spider_log): self.new_batch = CallLaterOnce(new_batch) self.new_batch.setErrback(self.error) self.consumption = CallLaterOnce(consume_incoming) self.consumption.setErrback(self.error) self.scheduling = CallLaterOnce(self.schedule) self.scheduling.setErrback(self.error) self.scoring_consumption = CallLaterOnce(consume_scoring) self.scoring_consumption.setErrback(self.error) self.no_batches = no_batches self.no_scoring_log = no_scoring_log self.no_spider_log = no_spider_log self.new_batch_delay = new_batch_delay def error(self, f): if isinstance(f, Failure): logger.error(f.value, exc_info=(f.type, f.value, f.getTracebackObject())) else: self.exception(f.value) return f def schedule(self, on_start=False): if on_start and not self.no_batches: self.new_batch.schedule(0) if not self.no_spider_log: self.consumption.schedule() if not self.no_batches: self.new_batch.schedule(self.new_batch_delay) if not self.no_scoring_log: self.scoring_consumption.schedule() self.scheduling.schedule(5.0)