def test_call_later_once(self):
self.called = 0
reactor = Clock()
call = CallLaterOnce(self.call_function, reactor=reactor)
call.schedule(delay=1)
reactor.advance(1)
assert self.called == 1
def test_call_later_once(self):
self.called = 0
reactor = Clock()
call = CallLaterOnce(self.call_function, reactor=reactor)
call.schedule(delay=1)
reactor.advance(1)
assert self.called == 1
def test_call_later_twice_already_scheduled(self):
self.called = 0
reactor = Clock()
call = CallLaterOnce(self.call_function, reactor=reactor)
call.schedule(delay=1)
call.schedule(delay=2)
reactor.advance(2)
assert self.called == 1
def test_call_later_twice_already_scheduled(self):
self.called = 0
reactor = Clock()
call = CallLaterOnce(self.call_function, reactor=reactor)
call.schedule(delay=1)
call.schedule(delay=2)
reactor.advance(2)
assert self.called == 1
def __init__(self, new_batch, consume_incoming, consume_scoring, no_batches, no_scoring_log,
new_batch_delay, no_spider_log):
self.new_batch = CallLaterOnce(new_batch)
self.new_batch.setErrback(self.error)
self.consumption = CallLaterOnce(consume_incoming)
self.consumption.setErrback(self.error)
self.scheduling = CallLaterOnce(self.schedule)
self.scheduling.setErrback(self.error)
self.scoring_consumption = CallLaterOnce(consume_scoring)
self.scoring_consumption.setErrback(self.error)
self.no_batches = no_batches
self.no_scoring_log = no_scoring_log
self.no_spider_log = no_spider_log
self.new_batch_delay = new_batch_delay
def __init__(self, new_batch, consume_incoming, consume_scoring, no_batches, no_scoring_log,
new_batch_delay, no_spider_log):
self.new_batch = CallLaterOnce(new_batch)
self.new_batch.setErrback(self.error)
self.consumption = CallLaterOnce(consume_incoming)
self.consumption.setErrback(self.error)
self.scheduling = CallLaterOnce(self.schedule)
self.scheduling.setErrback(self.error)
self.scoring_consumption = CallLaterOnce(consume_scoring)
self.scoring_consumption.setErrback(self.error)
self.no_batches = no_batches
self.no_scoring_log = no_scoring_log
self.no_spider_log = no_spider_log
self.new_batch_delay = new_batch_delay
def __init__(self, new_batch, consume_incoming, consume_scoring, no_batches, enable_strategy_worker,
new_batch_delay, no_incoming):
self.new_batch = CallLaterOnce(new_batch)
self.new_batch.setErrback(self.error)
self.consumption = CallLaterOnce(consume_incoming)
self.consumption.setErrback(self.error)
self.scheduling = CallLaterOnce(self.schedule)
self.scheduling.setErrback(self.error)
self.scoring_consumption = CallLaterOnce(consume_scoring)
self.scoring_consumption.setErrback(self.error)
self.disable_new_batches = no_batches
self.disable_scoring_consumption = not enable_strategy_worker
self.disable_incoming = no_incoming
self.new_batch_delay = new_batch_delay
class DBWorkerPeriodicComponent(DBWorkerBaseComponent):
def __init__(self, worker, settings, stop_event, *args, **kwargs):
super(DBWorkerPeriodicComponent, self).__init__(worker, settings, stop_event)
self.periodic_task = CallLaterOnce(self.run_and_reschedule)
self.periodic_task.setErrback(self.run_errback)
def schedule(self, delay=0):
self.periodic_task.schedule(delay)
def run_and_reschedule(self):
if not self.stopped:
self.run()
self.periodic_task.schedule()
def run_errback(self, failure):
self.logger.error(failure.getTraceback())
if not self.stopped:
self.periodic_task.schedule()
@property
def stopped(self):
return self.stop_event.is_set()
def __init__(self, worker, settings, stop_event, *args, **kwargs):
super(DBWorkerPeriodicComponent, self).__init__(worker, settings, stop_event)
self.periodic_task = CallLaterOnce(self.run_and_reschedule)
self.periodic_task.setErrback(self.run_errback)
class Slot(object):
def __init__(self, new_batch, consume_incoming, consume_scoring,
no_batches, enable_strategy_worker, new_batch_delay,
no_incoming):
self.new_batch = CallLaterOnce(new_batch)
self.new_batch.setErrback(self.error)
self.consumption = CallLaterOnce(consume_incoming)
self.consumption.setErrback(self.error)
self.scheduling = CallLaterOnce(self.schedule)
self.scheduling.setErrback(self.error)
self.scoring_consumption = CallLaterOnce(consume_scoring)
self.scoring_consumption.setErrback(self.error)
self.disable_new_batches = no_batches
self.disable_scoring_consumption = not enable_strategy_worker
self.disable_incoming = no_incoming
self.new_batch_delay = new_batch_delay
def error(self, f):
logger.exception(f.value)
return f
def schedule(self, on_start=False):
if on_start and not self.disable_new_batches:
self.new_batch.schedule(0)
if not self.disable_incoming:
self.consumption.schedule()
if not self.disable_new_batches:
self.new_batch.schedule(self.new_batch_delay)
if not self.disable_scoring_consumption:
self.scoring_consumption.schedule()
self.scheduling.schedule(5.0)
class Slot(object):
def __init__(self, new_batch, consume_incoming, consume_scoring, no_batches, enable_strategy_worker,
new_batch_delay, no_incoming):
self.new_batch = CallLaterOnce(new_batch)
self.new_batch.setErrback(self.error)
self.consumption = CallLaterOnce(consume_incoming)
self.consumption.setErrback(self.error)
self.scheduling = CallLaterOnce(self.schedule)
self.scheduling.setErrback(self.error)
self.scoring_consumption = CallLaterOnce(consume_scoring)
self.scoring_consumption.setErrback(self.error)
self.disable_new_batches = no_batches
self.disable_scoring_consumption = not enable_strategy_worker
self.disable_incoming = no_incoming
self.new_batch_delay = new_batch_delay
def error(self, f):
logger.exception(f.value)
return f
def schedule(self, on_start=False):
if on_start and not self.disable_new_batches:
self.new_batch.schedule(0)
if not self.disable_incoming:
self.consumption.schedule()
if not self.disable_new_batches:
self.new_batch.schedule(self.new_batch_delay)
if not self.disable_scoring_consumption:
self.scoring_consumption.schedule()
self.scheduling.schedule(5.0)
class Slot(object):
def __init__(self, new_batch, consume_incoming, consume_scoring,
no_batches, no_scoring_log, new_batch_delay, no_spider_log):
self.new_batch = CallLaterOnce(new_batch)
self.new_batch.setErrback(self.error)
self.consumption = CallLaterOnce(consume_incoming)
self.consumption.setErrback(self.error)
self.scheduling = CallLaterOnce(self.schedule)
self.scheduling.setErrback(self.error)
self.scoring_consumption = CallLaterOnce(consume_scoring)
self.scoring_consumption.setErrback(self.error)
self.no_batches = no_batches
self.no_scoring_log = no_scoring_log
self.no_spider_log = no_spider_log
self.new_batch_delay = new_batch_delay
def error(self, f):
logger.exception(f.value)
return f
def schedule(self, on_start=False):
if on_start and not self.no_batches:
self.new_batch.schedule(0)
if not self.no_spider_log:
self.consumption.schedule()
if not self.no_batches:
self.new_batch.schedule(self.new_batch_delay)
if not self.no_scoring_log:
self.scoring_consumption.schedule()
self.scheduling.schedule(5.0)
def cancel(self):
self.scheduling.cancel()
self.scoring_consumption.cancel()
self.new_batch.cancel()
self.consumption.cancel()
class Slot(object):
def __init__(self, new_batch, consume_incoming, consume_scoring,
no_batches, no_scoring_log, new_batch_delay, no_spider_log):
self.new_batch = CallLaterOnce(new_batch)
self.new_batch.setErrback(self.error)
self.consumption = CallLaterOnce(consume_incoming)
self.consumption.setErrback(self.error)
self.scheduling = CallLaterOnce(self.schedule)
self.scheduling.setErrback(self.error)
self.scoring_consumption = CallLaterOnce(consume_scoring)
self.scoring_consumption.setErrback(self.error)
self.no_batches = no_batches
self.no_scoring_log = no_scoring_log
self.no_spider_log = no_spider_log
self.new_batch_delay = new_batch_delay
def error(self, f):
if isinstance(f, Failure):
logger.error(f.value,
exc_info=(f.type, f.value, f.getTracebackObject()))
else:
self.exception(f.value)
return f
def schedule(self, on_start=False):
if on_start and not self.no_batches:
self.new_batch.schedule(0)
if not self.no_spider_log:
self.consumption.schedule()
if not self.no_batches:
self.new_batch.schedule(self.new_batch_delay)
if not self.no_scoring_log:
self.scoring_consumption.schedule()
self.scheduling.schedule(5.0)
