class TestProcessorWithStartedTask(unittest.TestCase): """We want to make sure administrative tools can mark a task as failed or completed without calling start, if the task is already storing its PID """ @mock.patch("redis.StrictRedis", autospec=True) @mock.patch("blueque.client.RedisQueue", autospec=True) def setUp(self, mock_redis_queue_class, mock_strict_redis): self.mock_redis_queue = mock_redis_queue_class.return_value self.mock_strict_redis = mock_strict_redis.from_url.return_value self.mock_strict_redis.hgetall.return_value = { "queue": "some.queue", "status": "reserved", "node": "host_1234", "pid": 4321, "parameters": "some parameters" } self.client = Client("redis://asdf:1234") self.task = self.client.get_task("some_task") self.processor = self.client.get_processor(self.task) def test_complete_marks_task_completed(self): self.processor.complete("some result") self.mock_redis_queue.complete.assert_called_with( "some_task", "host_1234", 4321, "some result") def test_fail_marks_task_failed(self): self.processor.fail("some error") self.mock_redis_queue.fail.assert_called_with("some_task", "host_1234", 4321, "some error")
class TestProcessorWithStartedTask(unittest.TestCase): """We want to make sure administrative tools can mark a task as failed or completed without calling start, if the task is already storing its PID """ @mock.patch("redis.StrictRedis", autospec=True) @mock.patch("blueque.client.RedisQueue", autospec=True) def setUp(self, mock_redis_queue_class, mock_strict_redis): self.mock_redis_queue = mock_redis_queue_class.return_value self.mock_strict_redis = mock_strict_redis.return_value self.mock_strict_redis.hgetall.return_value = { "queue": "some.queue", "status": "reserved", "node": "host_1234", "pid": 4321, "parameters": "some parameters", } self.client = Client(hostname="asdf", port=1234, db=0) self.task = self.client.get_task("some_task") self.processor = self.client.get_processor(self.task) def test_complete_marks_task_completed(self): self.processor.complete("some result") self.mock_redis_queue.complete.assert_called_with("some_task", "host_1234", 4321, "some result") def test_fail_marks_task_failed(self): self.processor.fail("some error") self.mock_redis_queue.fail.assert_called_with("some_task", "host_1234", 4321, "some error")
class TestQueue(unittest.TestCase): @mock.patch("redis.StrictRedis", autospec=True) @mock.patch("blueque.client.RedisQueue", autospec=True) def setUp(self, mock_redis_queue_class, mock_strict_redis): self.mock_strict_redis = mock_strict_redis self.mock_redis_queue_class = mock_redis_queue_class self.mock_redis_queue = mock_redis_queue_class.return_value self.client = Client("asdf", 1234, 0) self.queue = self.client.get_queue("some.queue") def test_name_passed_to_redis_queue(self): self.mock_redis_queue_class.assert_called_with( "some.queue", self.mock_strict_redis.return_value) def test_enqueue_enqueues_task(self): self.mock_redis_queue.enqueue.return_value = "task_id" task_id = self.queue.enqueue("the parameters") self.assertEqual("task_id", task_id) self.mock_redis_queue.enqueue.assert_called_with("the parameters") def test_schedule_schedules_task(self): self.mock_redis_queue.schedule.return_value = "task_id" task_id = self.queue.schedule("some parameters", 24.3) self.assertEqual("task_id", task_id) self.mock_redis_queue.schedule.assert_called_with("some parameters", 24.3) def test_enqueue_due_tasks_enqueues_due_tasks(self): self.queue.enqueue_due_tasks() self.mock_redis_queue.enqueue_due_tasks.assert_called_with() def test_delete_deletes_task(self): self.mock_strict_redis.return_value.hgetall.return_value = { "status": "complete", "queue": "some.queue" } task = self.client.get_task("some_task") self.queue.delete_task(task) self.mock_redis_queue.delete_task.assert_called_with("some_task", "complete") def test_delete_errors_on_wrong_queue(self): self.mock_strict_redis.return_value.hgetall.return_value = { "status": "complete", "queue": "other.queue" } task = self.client.get_task("some_task") with self.assertRaisesRegexp(ValueError, "Task some_task is not in queue some.queue"): self.queue.delete_task(task)
def setUp(self, mock_redis_class): self.mock_strict_redis = mock_redis_class.from_url.return_value self.task_callback = mock.Mock() self.client = Client("redis://asdf:1234") self.runner = forking_runner.ForkingRunner(self.client, "some.queue", self.task_callback)
def setUp(self, mock_redis_queue_class, mock_strict_redis): self.mock_strict_redis = mock_strict_redis.from_url.return_value self.mock_redis_queue_class = mock_redis_queue_class self.mock_redis_queue = mock_redis_queue_class.return_value self.client = Client("redis://asdf:1234") self.queue = self.client.get_queue("some.queue")
def setUp(self, mock_redis_queue_class, mock_strict_redis): self.mock_redis_queue = mock_redis_queue_class.return_value self.mock_strict_redis = mock_strict_redis.from_url.return_value self.mock_strict_redis.hgetall.return_value = { "queue": "some.queue", "status": "reserved", "node": "host_1234", "parameters": "some parameters" } self.client = Client("redis://asdf:1234") self.task = self.client.get_task("some_task") self.processor = self.client.get_processor(self.task)
def test_client_passes_additional_args_to_client(self, mock_redis_class): self.client = Client("redis://url", 4, socket_timeout=5) mock_redis_class.from_url.assert_called_with("redis://url", 4, decode_responses=True, socket_timeout=5)
def setUp(self, strict_redis_class): self.mock_strict_redis = strict_redis_class.return_value self.task_callback = mock.Mock() self.client = Client(hostname="asdf", port=1234, db=0) self.runner = forking_runner.ForkingRunner(self.client, "some.queue", self.task_callback)
def setUp(self, mock_redis_queue_class, mock_strict_redis): self.mock_strict_redis = mock_strict_redis self.mock_redis_queue_class = mock_redis_queue_class self.mock_redis_queue = mock_redis_queue_class.return_value self.client = Client("asdf", 1234, 0) self.queue = self.client.get_queue("some.queue")
class TestProcessor(unittest.TestCase): @mock.patch("redis.StrictRedis", autospec=True) @mock.patch("blueque.client.RedisQueue", autospec=True) def setUp(self, mock_redis_queue_class, mock_strict_redis): self.mock_redis_queue = mock_redis_queue_class.return_value self.mock_strict_redis = mock_strict_redis.from_url.return_value self.mock_strict_redis.hgetall.return_value = { "queue": "some.queue", "status": "reserved", "node": "host_1234", "parameters": "some parameters" } self.client = Client("redis://asdf:1234") self.task = self.client.get_task("some_task") self.processor = self.client.get_processor(self.task) def test_start_starts_processor(self): self.processor.start(4321) self.mock_redis_queue.start.assert_called_with("some_task", "host_1234", 4321) def test_complete_marks_task_completed(self): # Can't complete an unstarted process self.processor.start(4321) self.processor.complete("some result") self.mock_redis_queue.complete.assert_called_with( "some_task", "host_1234", 4321, "some result") def test_fail_marks_task_failed(self): # Can't complete an unstarted process self.processor.start(4321) self.processor.fail("some error") self.mock_redis_queue.fail.assert_called_with("some_task", "host_1234", 4321, "some error")
class TestProcessor(unittest.TestCase): @mock.patch("redis.StrictRedis", autospec=True) @mock.patch("blueque.client.RedisQueue", autospec=True) def setUp(self, mock_redis_queue_class, mock_strict_redis): self.mock_redis_queue = mock_redis_queue_class.return_value self.mock_strict_redis = mock_strict_redis.return_value self.mock_strict_redis.hgetall.return_value = { "queue": "some.queue", "status": "reserved", "node": "host_1234", "parameters": "some parameters" } self.client = Client(hostname="asdf", port=1234, db=0) self.task = self.client.get_task("some_task") self.processor = self.client.get_processor(self.task) def test_start_starts_processor(self): self.processor.start(4321) self.mock_redis_queue.start.assert_called_with("some_task", "host_1234", 4321) def test_complete_marks_task_completed(self): # Can't complete an unstarted process self.processor.start(4321) self.processor.complete("some result") self.mock_redis_queue.complete.assert_called_with( "some_task", "host_1234", 4321, "some result") def test_fail_marks_task_failed(self): # Can't complete an unstarted process self.processor.start(4321) self.processor.fail("some error") self.mock_redis_queue.fail.assert_called_with( "some_task", "host_1234", 4321, "some error")
def setUp(self, mock_redis_queue_class, mock_strict_redis): self.mock_redis_queue = mock_redis_queue_class.return_value self.mock_strict_redis = mock_strict_redis.return_value self.mock_strict_redis.hgetall.return_value = { "queue": "some.queue", "status": "reserved", "node": "host_1234", "parameters": "some parameters", } self.client = Client(hostname="asdf", port=1234, db=0) self.task = self.client.get_task("some_task") self.processor = self.client.get_processor(self.task)
class TestQueue(unittest.TestCase): @mock.patch("redis.StrictRedis", autospec=True) @mock.patch("blueque.client.RedisQueue", autospec=True) def setUp(self, mock_redis_queue_class, mock_strict_redis): self.mock_strict_redis = mock_strict_redis.from_url.return_value self.mock_redis_queue_class = mock_redis_queue_class self.mock_redis_queue = mock_redis_queue_class.return_value self.client = Client("redis://asdf:1234") self.queue = self.client.get_queue("some.queue") def test_name_passed_to_redis_queue(self): self.mock_redis_queue_class.assert_called_with("some.queue", self.mock_strict_redis) def test_enqueue_enqueues_task(self): self.mock_redis_queue.enqueue.return_value = "task_id" task_id = self.queue.enqueue("the parameters") self.assertEqual("task_id", task_id) self.mock_redis_queue.enqueue.assert_called_with("the parameters") def test_schedule_schedules_task(self): self.mock_redis_queue.schedule.return_value = "task_id" task_id = self.queue.schedule("some parameters", 24.3) self.assertEqual("task_id", task_id) self.mock_redis_queue.schedule.assert_called_with( "some parameters", 24.3) def test_enqueue_due_tasks_enqueues_due_tasks(self): self.queue.enqueue_due_tasks() self.mock_redis_queue.enqueue_due_tasks.assert_called_with() def test_delete_deletes_task(self): self.mock_strict_redis.hgetall.return_value = { "status": "complete", "queue": "some.queue" } task = self.client.get_task("some_task") self.queue.delete_task(task) self.mock_redis_queue.delete_task.assert_called_with( "some_task", "complete") def test_delete_errors_on_wrong_queue(self): self.mock_strict_redis.hgetall.return_value = { "status": "complete", "queue": "other.queue" } task = self.client.get_task("some_task") with self.assertRaisesRegexp( ValueError, "Task some_task is not in queue some.queue"): self.queue.delete_task(task)
def setUp(self, redis_class): self.mock_redis = redis_class.return_value self.client = Client("asdf", 1234, 0)
class TestForkingRunner(unittest.TestCase): @mock.patch("redis.StrictRedis", autospec=True) def setUp(self, mock_redis_class): self.mock_strict_redis = mock_redis_class.from_url.return_value self.task_callback = mock.Mock() self.client = Client("redis://*****:*****@mock.patch("logging.info") @mock.patch("os.fork", return_value=1234) @mock.patch("os.waitpid", return_value=(1234, 0)) def test_run_starts_unstarted_orphan(self, mock_waitpid, mock_fork, mock_info, redis_queue_class): with mock.patch.object(self.client, 'get_listener') as mock_get_listener: mock_listener = mock_get_listener.return_value mock_listener.listen.side_effect = BreakLoop() mock_listener.claim_orphan.side_effect = [ self._get_task(status="reserved"), None ] try: self.runner.run() except BreakLoop: pass mock_get_listener.assert_called_with("some.queue") mock_listener.claim_orphan.assert_called_with() mock_fork.assert_has_calls([mock.call()]) mock_waitpid.assert_has_calls([mock.call(1234, 0)]) mock_info.assert_has_calls([ mock.call("Forked task some_task to pid 1234"), mock.call("Forked task some_task exited with status 0") ]) @mock.patch("logging.info") @mock.patch("os.kill", side_effect=[None, None, OSError]) @mock.patch("time.sleep") def test_run_watches_started_orphan(self, mock_sleep, mock_kill, mock_info, redis_queue_class): with mock.patch.object(self.client, 'get_listener') as mock_get_listener: mock_listener = mock_get_listener.return_value mock_listener.listen.side_effect = BreakLoop() mock_listener.claim_orphan.side_effect = [ self._get_task(status="started", pid="1111"), None ] try: self.runner.run() except BreakLoop: pass mock_get_listener.assert_called_with("some.queue") mock_listener.claim_orphan.assert_called_with() mock_kill.assert_has_calls( [mock.call(1111, 0), mock.call(1111, 0), mock.call(1111, 0)]) mock_sleep.assert_has_calls([mock.call(0.1), mock.call(0.1)]) @mock.patch("logging.info") @mock.patch("os.kill", side_effect=[OSError, OSError]) @mock.patch("time.sleep") def test_run_exhausts_all_orphans(self, mock_sleep, mock_kill, mock_info, redis_queue_class): with mock.patch.object(self.client, 'get_listener') as mock_get_listener: mock_listener = mock_get_listener.return_value mock_listener.listen.side_effect = BreakLoop() mock_listener.claim_orphan.side_effect = [ self._get_task(status="started", pid="1111"), self._get_task(status="started", pid="2222"), None ] try: self.runner.run() except BreakLoop: pass mock_get_listener.assert_called_with("some.queue") mock_listener.claim_orphan.assert_has_calls( [mock.call(), mock.call(), mock.call()]) mock_kill.assert_has_calls([mock.call(1111, 0), mock.call(2222, 0)]) @mock.patch("logging.info") @mock.patch("os.fork", return_value=1234) @mock.patch("os.waitpid", return_value=(1234, 0)) def test_run_listens_for_and_forks_task(self, mock_waitpid, mock_fork, mock_info, redis_queue_class): mock_queue = redis_queue_class.return_value mock_queue.get_listeners.return_value = [] mock_queue.dequeue.side_effect = ["some_task", BreakLoop()] try: # Don't want a real thread. self.runner.run() except BreakLoop: pass redis_queue_class.assert_called_with("some.queue", self.mock_strict_redis) mock_queue.get_listeners.assert_called_with() mock_fork.assert_has_calls([mock.call()]) mock_waitpid.assert_has_calls([mock.call(1234, 0)]) mock_info.assert_has_calls([ mock.call("Forked task some_task to pid 1234"), mock.call("Forked task some_task exited with status 0") ]) @mock.patch("os.fork", side_effect=[1234, 4321]) @mock.patch("os.waitpid", side_effect=[(1234, 0), (4321, 0)]) def test_run_keeps_running_tasks(self, mock_waitpid, mock_fork, redis_queue_class): mock_queue = redis_queue_class.return_value mock_queue.dequeue.side_effect = [ "some_task", "other_task", BreakLoop() ] try: # Don't want a real thread. self.runner.run() except BreakLoop: pass redis_queue_class.assert_called_with("some.queue", self.mock_strict_redis) mock_fork.assert_has_calls([mock.call(), mock.call()]) mock_waitpid.assert_has_calls([mock.call(1234, 0), mock.call(4321, 0)]) @mock.patch("os.fork", return_value=1234) def test_fork_task_returns_pid_in_parent(self, mock_fork, redis_queue_class): task = self._get_task() pid = self.runner.fork_task(task) self.assertEqual(1234, pid) @mock.patch("logging.shutdown") @mock.patch("sys.stdout", wraps=sys.stdout) @mock.patch("sys.stderr", wraps=sys.stderr) @mock.patch("random.seed") @mock.patch("os.getpid", return_value=2222) @mock.patch("os.setsid") @mock.patch("os.fork", return_value=0) @mock.patch("os._exit") def test_fork_task_runs_task_in_child(self, mock_exit, mock_fork, mock_setsid, _, mock_seed, mock_stderr, mock_stdout, mock_log_shutdown, redis_queue_class): mock_queue = redis_queue_class.return_value self.task_callback.return_value = "some result" task = self._get_task() self.runner.fork_task(task) mock_seed.assert_called_with() mock_setsid.assert_called_with() mock_queue.start.assert_called_with("some_task", "some.host_1111", 2222) self.task_callback.assert_called_with(task) mock_queue.complete.assert_called_with("some_task", "some.host_1111", 2222, "some result") mock_log_shutdown.assert_called_with() mock_stdout.flush.assert_called_with() mock_stderr.flush.assert_called_with() mock_exit.assert_called_with(0) @mock.patch("logging.shutdown") @mock.patch("sys.stdout", wraps=sys.stdout) @mock.patch("sys.stderr", wraps=sys.stderr) @mock.patch("os.getpid", return_value=2222) @mock.patch("os.setsid") @mock.patch("os.fork", return_value=0) @mock.patch("os._exit") def test_fork_task_fails_task_on_exception(self, mock_exit, mock_fork, mock_setsid, _, mock_stderr, mock_stdout, mock_log_shutdown, redis_queue_class): mock_queue = redis_queue_class.return_value callback_exception = Exception("some error") self.task_callback.side_effect = callback_exception task = self._get_task() self.runner.fork_task(task) mock_setsid.assert_called_with() mock_queue.start.assert_called_with("some_task", "some.host_1111", 2222) self.task_callback.assert_called_with(task) mock_queue.fail.assert_called_with("some_task", "some.host_1111", 2222, str(callback_exception)) mock_log_shutdown.assert_called_with() mock_stdout.flush.assert_called_with() mock_stderr.flush.assert_called_with() mock_exit.assert_called_with(0)
def test_client_connects_with_requested_information( self, mock_redis_class): self.client = Client("redis://url") mock_redis_class.from_url.assert_called_with("redis://url", decode_responses=True)
class TestTask(unittest.TestCase): @mock.patch("redis.StrictRedis", autospec=True) def setUp(self, redis_class): self.mock_redis = redis_class.return_value self.client = Client("asdf", 1234, 0) def test_can_get_task_with_all_attributes(self): self.mock_redis.hgetall.return_value = FULL_TASK_DATA task = self.client.get_task("some_task") self.mock_redis.hgetall.assert_called_with("blueque_task_some_task") self.assertEqual("some_task", task.id) self.assertEqual("complete", task.status) self.assertEqual("some.queue", task.queue) self.assertEqual("some parameters", task.parameters) self.assertEqual("a result", task.result) self.assertEqual("no error", task.error) self.assertEqual("some_node", task.node) self.assertEqual(1234, task.pid) self.assertEqual(1234.5, task.created) self.assertEqual(4567.89, task.updated) def test_cannot_set_properties(self): self.mock_redis.hgetall.return_value = FULL_TASK_DATA task = self.client.get_task("some_task") with self.assertRaises(AttributeError): task.id = "foo" with self.assertRaises(AttributeError): task.status = "foo" with self.assertRaises(AttributeError): task.queue = "foo" with self.assertRaises(AttributeError): task.parameters = "foo" with self.assertRaises(AttributeError): task.result = "foo" with self.assertRaises(AttributeError): task.error = "foo" with self.assertRaises(AttributeError): task.node = "foo" with self.assertRaises(AttributeError): task.pid = 4321 with self.assertRaises(AttributeError): task.created = 1.2 with self.assertRaises(AttributeError): task.updated = 2.3 def test_missing_attributes_are_none(self): self.mock_redis.hgetall.return_value = {} task = self.client.get_task("some_task") self.assertEqual(None, task.status) self.assertEqual(None, task.queue) self.assertEqual(None, task.parameters) self.assertEqual(None, task.result) self.assertEqual(None, task.error) self.assertEqual(None, task.node) self.assertEqual(None, task.pid) self.assertEqual(None, task.created) self.assertEqual(None, task.updated)
class TestForkingRunner(unittest.TestCase): @mock.patch("redis.StrictRedis", autospec=True) def setUp(self, strict_redis_class): self.mock_strict_redis = strict_redis_class.return_value self.task_callback = mock.Mock() self.client = Client(hostname="asdf", port=1234, db=0) self.runner = forking_runner.ForkingRunner(self.client, "some.queue", self.task_callback) def _get_task(self, **kwargs): task_data = { "status": "reserved", "parameters": "some params", "node": "some.host_1111" } task_data.update(kwargs) self.mock_strict_redis.hgetall.return_value = task_data return self.client.get_task("some_task") @mock.patch("logging.info") @mock.patch("os.fork", return_value=1234) @mock.patch("os.waitpid", return_value=(1234, 0)) def test_run_starts_unstarted_orphan( self, mock_waitpid, mock_fork, mock_info, redis_queue_class): with mock.patch.object(self.client, 'get_listener') as mock_get_listener: mock_listener = mock_get_listener.return_value mock_listener.listen.side_effect = BreakLoop() mock_listener.claim_orphan.side_effect = [self._get_task(status="reserved"), None] try: self.runner.run() except BreakLoop: pass mock_get_listener.assert_called_with("some.queue") mock_listener.claim_orphan.assert_called_with() mock_fork.assert_has_calls([mock.call()]) mock_waitpid.assert_has_calls([mock.call(1234, 0)]) mock_info.assert_has_calls([ mock.call("Forked task some_task to pid 1234"), mock.call("Forked task some_task exited with status 0") ]) @mock.patch("logging.info") @mock.patch("os.kill", side_effect=[None, None, OSError]) @mock.patch("time.sleep") def test_run_watches_started_orphan( self, mock_sleep, mock_kill, mock_info, redis_queue_class): with mock.patch.object(self.client, 'get_listener') as mock_get_listener: mock_listener = mock_get_listener.return_value mock_listener.listen.side_effect = BreakLoop() mock_listener.claim_orphan.side_effect = [ self._get_task(status="started", pid="1111"), None] try: self.runner.run() except BreakLoop: pass mock_get_listener.assert_called_with("some.queue") mock_listener.claim_orphan.assert_called_with() mock_kill.assert_has_calls([ mock.call(1111, 0), mock.call(1111, 0), mock.call(1111, 0)]) mock_sleep.assert_has_calls([mock.call(0.1), mock.call(0.1)]) @mock.patch("logging.info") @mock.patch("os.kill", side_effect=[OSError, OSError]) @mock.patch("time.sleep") def test_run_exhausts_all_orphans( self, mock_sleep, mock_kill, mock_info, redis_queue_class): with mock.patch.object(self.client, 'get_listener') as mock_get_listener: mock_listener = mock_get_listener.return_value mock_listener.listen.side_effect = BreakLoop() mock_listener.claim_orphan.side_effect = [ self._get_task(status="started", pid="1111"), self._get_task(status="started", pid="2222"), None] try: self.runner.run() except BreakLoop: pass mock_get_listener.assert_called_with("some.queue") mock_listener.claim_orphan.assert_has_calls([mock.call(), mock.call(), mock.call()]) mock_kill.assert_has_calls([ mock.call(1111, 0), mock.call(2222, 0)]) @mock.patch("logging.info") @mock.patch("os.fork", return_value=1234) @mock.patch("os.waitpid", return_value=(1234, 0)) def test_run_listens_for_and_forks_task( self, mock_waitpid, mock_fork, mock_info, redis_queue_class): mock_queue = redis_queue_class.return_value mock_queue.get_listeners.return_value = [] mock_queue.dequeue.side_effect = ["some_task", BreakLoop()] try: # Don't want a real thread. self.runner.run() except BreakLoop: pass redis_queue_class.assert_called_with("some.queue", self.mock_strict_redis) mock_queue.get_listeners.assert_called_with() mock_fork.assert_has_calls([mock.call()]) mock_waitpid.assert_has_calls([mock.call(1234, 0)]) mock_info.assert_has_calls([ mock.call("Forked task some_task to pid 1234"), mock.call("Forked task some_task exited with status 0") ]) @mock.patch("os.fork", side_effect=[1234, 4321]) @mock.patch("os.waitpid", side_effect=[(1234, 0), (4321, 0)]) def test_run_keeps_running_tasks(self, mock_waitpid, mock_fork, redis_queue_class): mock_queue = redis_queue_class.return_value mock_queue.dequeue.side_effect = ["some_task", "other_task", BreakLoop()] try: # Don't want a real thread. self.runner.run() except BreakLoop: pass redis_queue_class.assert_called_with("some.queue", self.mock_strict_redis) mock_fork.assert_has_calls([mock.call(), mock.call()]) mock_waitpid.assert_has_calls([mock.call(1234, 0), mock.call(4321, 0)]) @mock.patch("os.fork", return_value=1234) def test_fork_task_returns_pid_in_parent(self, mock_fork, redis_queue_class): task = self._get_task() pid = self.runner.fork_task(task) self.assertEqual(1234, pid) @mock.patch("logging.shutdown") @mock.patch("sys.stdout", wraps=sys.stdout) @mock.patch("sys.stderr", wraps=sys.stderr) @mock.patch("random.seed") @mock.patch("os.getpid", return_value=2222) @mock.patch("os.setsid") @mock.patch("os.fork", return_value=0) @mock.patch("os._exit") def test_fork_task_runs_task_in_child( self, mock_exit, mock_fork, mock_setsid, _, mock_seed, mock_stderr, mock_stdout, mock_log_shutdown, redis_queue_class): mock_queue = redis_queue_class.return_value self.task_callback.return_value = "some result" task = self._get_task() self.runner.fork_task(task) mock_seed.assert_called_with() mock_setsid.assert_called_with() mock_queue.start.assert_called_with("some_task", "some.host_1111", 2222) self.task_callback.assert_called_with(task) mock_queue.complete.assert_called_with("some_task", "some.host_1111", 2222, "some result") mock_log_shutdown.assert_called_with() mock_stdout.flush.assert_called_with() mock_stderr.flush.assert_called_with() mock_exit.assert_called_with(0) @mock.patch("logging.shutdown") @mock.patch("sys.stdout", wraps=sys.stdout) @mock.patch("sys.stderr", wraps=sys.stderr) @mock.patch("os.getpid", return_value=2222) @mock.patch("os.setsid") @mock.patch("os.fork", return_value=0) @mock.patch("os._exit") def test_fork_task_fails_task_on_exception( self, mock_exit, mock_fork, mock_setsid, _, mock_stderr, mock_stdout, mock_log_shutdown, redis_queue_class): mock_queue = redis_queue_class.return_value callback_exception = Exception("some error") self.task_callback.side_effect = callback_exception task = self._get_task() self.runner.fork_task(task) mock_setsid.assert_called_with() mock_queue.start.assert_called_with("some_task", "some.host_1111", 2222) self.task_callback.assert_called_with(task) mock_queue.fail.assert_called_with( "some_task", "some.host_1111", 2222, str(callback_exception)) mock_log_shutdown.assert_called_with() mock_stdout.flush.assert_called_with() mock_stderr.flush.assert_called_with() mock_exit.assert_called_with(0)
# This hasn't actually been run; it's just a scratch-pad, for now, to # figure out what API we want from __future__ import print_function from blueque import Client from blueque import forking_runner import time def do_work(task): print(task.id, task.parameters) time.sleep(1000) return "result" if __name__ == "__main__": client = Client("redis://localhost") forking_runner.run(client, "some.queue", do_work, 4)
class TestTask(unittest.TestCase): @mock.patch("redis.StrictRedis", autospec=True) def setUp(self, redis_class): self.mock_redis = redis_class.from_url.return_value self.client = Client("redis://asdf:1234") def test_can_get_task_with_all_attributes(self): self.mock_redis.hgetall.return_value = FULL_TASK_DATA task = self.client.get_task("some_task") self.mock_redis.hgetall.assert_called_with("blueque_task_some_task") self.assertEqual("some_task", task.id) self.assertEqual("complete", task.status) self.assertEqual("some.queue", task.queue) self.assertEqual("some parameters", task.parameters) self.assertEqual("a result", task.result) self.assertEqual("no error", task.error) self.assertEqual("some_node", task.node) self.assertEqual(1234, task.pid) self.assertEqual(1234.5, task.created) self.assertEqual(4567.89, task.updated) def test_cannot_set_properties(self): self.mock_redis.hgetall.return_value = FULL_TASK_DATA task = self.client.get_task("some_task") with self.assertRaises(AttributeError): task.id = "foo" with self.assertRaises(AttributeError): task.status = "foo" with self.assertRaises(AttributeError): task.queue = "foo" with self.assertRaises(AttributeError): task.parameters = "foo" with self.assertRaises(AttributeError): task.result = "foo" with self.assertRaises(AttributeError): task.error = "foo" with self.assertRaises(AttributeError): task.node = "foo" with self.assertRaises(AttributeError): task.pid = 4321 with self.assertRaises(AttributeError): task.created = 1.2 with self.assertRaises(AttributeError): task.updated = 2.3 def test_missing_attributes_are_none(self): self.mock_redis.hgetall.return_value = {} task = self.client.get_task("some_task") self.assertEqual(None, task.status) self.assertEqual(None, task.queue) self.assertEqual(None, task.parameters) self.assertEqual(None, task.result) self.assertEqual(None, task.error) self.assertEqual(None, task.node) self.assertEqual(None, task.pid) self.assertEqual(None, task.created) self.assertEqual(None, task.updated)
def setUp(self, redis_class): self.mock_redis = redis_class.from_url.return_value self.client = Client("redis://asdf:1234")
class TestListener(unittest.TestCase): @mock.patch("socket.getfqdn", return_value="somehost.example.com") @mock.patch("os.getpid", return_value=2314) @mock.patch("redis.StrictRedis", autospec=True) @mock.patch("blueque.client.RedisQueue", autospec=True) def setUp(self, mock_redis_queue_class, mock_strict_redis, _, __): self.mock_strict_redis = mock_strict_redis self.mock_redis_queue_class = mock_redis_queue_class self.mock_redis_queue = mock_redis_queue_class.return_value self.client = Client("asdf", 1234, 0) self.listener = self.client.get_listener("some.queue") def test_listener_adds_itself(self): self.mock_redis_queue.add_listener.assert_called_with("somehost.example.com_2314") def test_listener_calls_callback_when_task_in_queue(self): self.mock_strict_redis.return_value.hgetall.return_value = { "parameters": "some parameters" } self.mock_redis_queue.dequeue.side_effect = ["some_task"] task = self.listener.listen() self.mock_redis_queue.dequeue.assert_called_with("somehost.example.com_2314") self.assertEqual("some_task", task.id) self.assertEqual("some parameters", task.parameters) @mock.patch("time.sleep", autospec=True) def test_listener_sleeps_when_no_task_available(self, mock_sleep): self.mock_redis_queue.dequeue.side_effect = [None, "some_task"] task = self.listener.listen() self.mock_redis_queue.dequeue.assert_has_calls( [mock.call("somehost.example.com_2314"), mock.call("somehost.example.com_2314")]) mock_sleep.assert_has_calls([mock.call(1)]) self.assertEqual("some_task", task.id) def test_claim_orphan_returns_none_when_there_are_no_listeners(self): self.mock_redis_queue.get_listeners.return_value = [] claimed = self.listener.claim_orphan() self.mock_redis_queue.get_listeners.assert_called_with() self.assertIsNone(claimed) @mock.patch("os.kill") def test_claim_orphan_returns_none_when_there_are_no_listeners_on_this_node(self, mock_kill): self.mock_redis_queue.get_listeners.return_value = ["other-host_4321"] claimed = self.listener.claim_orphan() self.mock_redis_queue.get_listeners.assert_called_with() self.assertIsNone(claimed) mock_kill.assert_not_called() @mock.patch("os.kill") def test_claim_orphan_returns_none_when_listener_is_self(self, mock_kill): self.mock_redis_queue.get_listeners.return_value = ["somehost.example.com_2314"] claimed = self.listener.claim_orphan() self.mock_redis_queue.get_listeners.assert_called_with() self.assertIsNone(claimed) mock_kill.assert_not_called() @mock.patch("os.kill", return_value=None) def test_claim_orphan_returns_none_when_listener_is_running(self, mock_kill): self.mock_redis_queue.get_listeners.return_value = ["somehost.example.com_4321"] claimed = self.listener.claim_orphan() self.mock_redis_queue.get_listeners.assert_called_with() self.assertIsNone(claimed) mock_kill.assert_called_with(4321, 0) @mock.patch("os.kill", side_effect=OSError) def test_claim_orphan_returns_none_when_orphan_claimed(self, mock_kill): self.mock_redis_queue.get_listeners.return_value = ["somehost.example.com_4321"] self.mock_redis_queue.remove_listener.return_value = 0 claimed = self.listener.claim_orphan() self.mock_redis_queue.get_listeners.assert_called_with() self.mock_redis_queue.remove_listener.assert_called_with("somehost.example.com_4321") mock_kill.assert_called_with(4321, 0) self.assertIsNone(claimed) @mock.patch("os.kill", side_effect=OSError) def test_claim_orphan_returns_none_when_no_tasks_reserved(self, mock_kill): self.mock_redis_queue.get_listeners.return_value = ["somehost.example.com_4321"] self.mock_redis_queue.remove_listener.return_value = 1 self.mock_redis_queue.reclaim_task.return_value = None claimed = self.listener.claim_orphan() self.mock_redis_queue.get_listeners.assert_called_with() self.mock_redis_queue.remove_listener.assert_called_with("somehost.example.com_4321") mock_kill.assert_called_with(4321, 0) self.mock_redis_queue.reclaim_task.assert_called_with( "somehost.example.com_4321", "somehost.example.com_2314") self.assertIsNone(claimed) @mock.patch("os.kill", side_effect=OSError) def test_claim_orphan_returns_task_when_reclaimed(self, mock_kill): self.mock_redis_queue.get_listeners.return_value = ["somehost.example.com_4321"] self.mock_redis_queue.remove_listener.return_value = 1 self.mock_redis_queue.reclaim_task.return_value = "some_task" self.mock_strict_redis.return_value.hgetall.return_value = { "parameters": "some parameters" } claimed = self.listener.claim_orphan() self.mock_redis_queue.get_listeners.assert_called_with() self.mock_redis_queue.remove_listener.assert_called_with("somehost.example.com_4321") mock_kill.assert_called_with(4321, 0) self.mock_redis_queue.reclaim_task.assert_called_with( "somehost.example.com_4321", "somehost.example.com_2314") self.mock_strict_redis.return_value.hgetall.assert_called_with("blueque_task_some_task") self.assertIsNotNone(claimed) self.assertEqual("some parameters", claimed.parameters)