def test_file_args_task(self): # Use a temp file to communicate since we're forking. tmpfile = tempfile.NamedTemporaryFile() worker = Worker(self.tiger) self.tiger.delay(file_args_task, args=(tmpfile.name, )) queues = self._ensure_queues(queued={'default': 1}) task = queues['queued']['default'][0] assert task['func'] == 'tests.tasks:file_args_task' worker.run(once=True) self._ensure_queues(queued={'default': 0}) json_data = tmpfile.read().decode('utf8') assert json.loads(json_data) == {'args': [], 'kwargs': {}} tmpfile.seek(0) self.tiger.delay(file_args_task, args=(tmpfile.name, 123, 'args'), kwargs={'more': [1, 2, 3]}) self._ensure_queues(queued={'default': 1}) worker.run(once=True) self._ensure_queues(queued={'default': 0}) json_data = tmpfile.read().decode('utf8') assert json.loads(json_data) == { 'args': [123, 'args'], 'kwargs': { 'more': [1, 2, 3] } }
def test_file_args_task(self): # Use a temp file to communicate since we're forking. tmpfile = tempfile.NamedTemporaryFile() worker = Worker(self.tiger) self.tiger.delay(file_args_task, args=(tmpfile.name,)) queues = self._ensure_queues(queued={'default': 1}) task = queues['queued']['default'][0] assert task['func'] == 'tests.tasks:file_args_task' worker.run(once=True) self._ensure_queues(queued={'default': 0}) json_data = tmpfile.read().decode('utf8') assert json.loads(json_data) == { 'args': [], 'kwargs': {} } tmpfile.seek(0) self.tiger.delay(file_args_task, args=(tmpfile.name, 123, 'args'), kwargs={'more': [1, 2, 3]}) self._ensure_queues(queued={'default': 1}) worker.run(once=True) self._ensure_queues(queued={'default': 0}) json_data = tmpfile.read().decode('utf8') assert json.loads(json_data) == { 'args': [123, 'args'], 'kwargs': {'more': [1, 2, 3]} }
def external_worker(n=None): """ Runs a worker. To be used with multiprocessing.Pool.map. """ tiger = get_tiger() worker = Worker(tiger) worker.run(once=True)
def test_max_workers(self): """Test Single Worker Queue.""" # Queue three tasks for i in range(0, 3): task = Task(self.tiger, long_task_ok, queue='a') task.delay() self._ensure_queues(queued={'a': 3}) # Start two workers and wait until they start processing. worker1 = Process(target=external_worker, kwargs={'max_workers_per_queue': 2}) worker2 = Process(target=external_worker, kwargs={'max_workers_per_queue': 2}) worker1.start() worker2.start() # Wait for both tasks to start wait_for_long_task() wait_for_long_task() # Verify they both are active self._ensure_queues(active={'a': 2}, queued={'a': 1}) # This worker should fail to get the queue lock and exit immediately worker = Worker(self.tiger) worker.max_workers_per_queue = 2 worker.run(once=True, force_once=True) self._ensure_queues(active={'a': 2}, queued={'a': 1}) # Wait for external workers worker1.join() worker2.join()
def test_max_workers(self): """Test Single Worker Queue.""" # Queue three tasks for i in range(0, 3): task = Task(self.tiger, long_task_killed, queue='a') task.delay() self._ensure_queues(queued={'a': 3}) # Start two workers and wait until they start processing. worker1 = Process(target=external_worker, kwargs={'max_workers_per_queue': 2}) worker2 = Process(target=external_worker, kwargs={'max_workers_per_queue': 2}) worker1.start() worker2.start() time.sleep(DELAY) # This worker should fail to get the queue lock and exit immediately worker = Worker(self.tiger) worker.max_workers_per_queue = 2 worker.run(once=True, force_once=True) self._ensure_queues(active={'a': 2}, queued={'a': 1}) # Wait for external workers worker1.join() worker2.join()
def external_worker(n=None, patch_config=None, max_workers_per_queue=None): """ Runs a worker. To be used with multiprocessing.Pool.map. """ tiger = get_tiger() if patch_config: tiger.config.update(patch_config) worker = Worker(tiger) if max_workers_per_queue is not None: worker.max_workers_per_queue = max_workers_per_queue worker.run(once=True, force_once=True)
def external_worker(n=None, patch_config=None, max_workers_per_queue=None): """ Runs a worker. To be used with multiprocessing.Pool.map. """ tiger = get_tiger() if patch_config: tiger.config.update(patch_config) worker = Worker(tiger) if max_workers_per_queue is not None: worker.max_workers_per_queue = max_workers_per_queue worker.run(once=True)
def test_queue_system_lock(self): """Test queue system lock.""" with FreezeTime(datetime.datetime(2014, 1, 1)): # Queue three tasks for i in range(0, 3): task = Task(self.tiger, long_task_ok, queue='a') task.delay() self._ensure_queues(queued={'a': 3}) # Ensure we can process one worker = Worker(self.tiger) worker.max_workers_per_queue = 2 worker.run(once=True, force_once=True) self._ensure_queues(queued={'a': 2}) # Set system lock so no processing should occur for 10 seconds self.tiger.set_queue_system_lock('a', 10) lock_timeout = self.tiger.get_queue_system_lock('a') assert lock_timeout == time.time() + 10 # Confirm tasks don't get processed within the system lock timeout with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 9)): worker = Worker(self.tiger) worker.max_workers_per_queue = 2 worker.run(once=True, force_once=True) self._ensure_queues(queued={'a': 2}) # 10 seconds in the future the lock should have expired with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 10)): worker = Worker(self.tiger) worker.max_workers_per_queue = 2 worker.run(once=True, force_once=True) self._ensure_queues(queued={'a': 1})