def setUp(self):
self.clock = Clock()
self.obj = ModifiedObject()
self.sc = ScheduledCall(self.obj.func,
clock=self.clock,
*self.default_args,
**self.default_kwargs)
def __init__(self, download_handler, concurrency, delay, randomize_delay,
clock=None):
self.download_handler = download_handler
self.concurrency = concurrency
self.delay = delay
self.randomize_delay = randomize_delay
self.in_progress = set() # request waiting to be downloaded
self.transferring = set() # requests being downloaded (subset of `in_progress`)
self.last_download_time = 0
self.queue = MemoryQueue() # queue of (request, deferred)
# clock is used in unittests
self.clock = clock or reactor
self.delayed_processing = ScheduledCall(self._process, clock=self.clock)
def __init__(self, settings, project, command_invoked='', clock=None):
'''Constructor of Engine should be very lightweight, so that things
can be easily unittested. For any more complicated initialization
use `setup()`.
'''
self.settings = settings
self.project = project
self.spiders = SpiderManager(settings)
self.stop_if_idle = True
self.initialized = False # True, when `setup()` has been called
# name of the command invoking the engine. E.g. `crawl`, `shell`, etc.
self.command_invoked = command_invoked
self.spider = None
self.pending_requests = 0
self.running = False
self.paused = False
# clock is used in unittests
self.clock = clock or reactor
self.processing = ScheduledCall(self._process_queue, clock=self.clock)
def test_init(self):
# test initializing ScheduledCall without overriding its clock
sc = ScheduledCall(self.obj.func, *self.default_args,
**self.default_kwargs)
sc.schedule()
sc.cancel()
