def test_multiple_submissions(mocker, docker_python): sched = QueueScheduler() mocker.spy(sched, 'on_update') with Running(sched, name='test-scheduler'): results = [None] * 3 for i in range(3): task = PythonTask(id=TaskID(value='test-python-task-{}'.format(i)), fn=sum, args=[[1, 10, i]], name='test-python-task-name', resources=[Cpus(0.1), Mem(64), Disk(0)]) results[i] = sched.submit(task) sched.wait() # block until all tasks finishes assert [r.get() for r in results] == [11, 12, 13]
def test_sequential_execution(mocker, docker_python): sched = QueueScheduler() mocker.spy(sched, 'on_update') with Running(sched, name='test-scheduler'): tasks = [] for i in range(3): task = PythonTask(id=TaskID(value='test-python-task-{}'.format(i)), fn=sum, args=[[1, 10, i]], name='test-python-task-name', resources=[Cpus(0.1), Mem(64), Disk(0)]) sched.submit(task) tasks.append(task) sched.wait() assert task.status.data == 11 + i
def test_docker_python(mocker, docker_python): sched = QueueScheduler() mocker.spy(sched, 'on_update') with Running(sched, name='test-scheduler'): sched.submit(docker_python) sched.wait() # block until all tasks finishes calls = sched.on_update.call_args_list assert len(calls) == 2 args, kwargs = calls[0] assert args[1].task_id.value == 'test-python-task-id' assert args[1].state == 'TASK_RUNNING' args, kwargs = calls[1] assert args[1].task_id.value == 'test-python-task-id' assert args[1].state == 'TASK_FINISHED'
def test_parallel_execution(mocker, docker_python): sched = QueueScheduler() mocker.spy(sched, 'on_update') with Running(sched, name='test-scheduler'): tasks = [] for i in range(3): task = PythonTask(id=TaskID(value='test-python-task-{}'.format(i)), fn=sum, args=[[1, 10, i]], name='test-python-task-name', resources=[Cpus(0.1), Mem(64), Disk(0)]) sched.submit(task) tasks.append(task) sched.wait() # block until all tasks finishes assert [t.status.data for t in tasks] == [11, 12, 13]
def test_docker_python_exception(): sched = QueueScheduler() def error(): raise TypeError('Dummy exception on executor side!') task = PythonTask(id=TaskID(value='test-python-task-id'), fn=error, name='test-python-task-name', resources=[Cpus(0.1), Mem(64), Disk(0)]) with Running(sched, name='test-scheduler'): sched.submit(task) sched.wait() assert task.status.has_failed() assert isinstance(task.status.exception, RemoteException) assert isinstance(task.status.exception, TypeError)
def test_docker_python(mocker, docker_python): sched = QueueScheduler() mocker.spy(sched, 'on_update') with Running(sched, name='test-scheduler'): sched.submit(docker_python) sched.wait() # block until all tasks finishes calls = sched.on_update.call_args_list assert len(calls) == 2 args, kwargs = calls[0] assert args[1].task_id.value == 'test-python-task-id' assert args[1].state == 'TASK_RUNNING' args, kwargs = calls[1] assert args[1].task_id.value == 'test-python-task-id' assert args[1].state == 'TASK_FINISHED'
def test_scheduler_retries(mocker): task = PythonTask(id=TaskID(value='non-existing-docker-image'), name='test', fn=lambda: range(int(10e10)), docker='pina/sen', resources=[Cpus(0.1), Mem(128), Disk(0)]) sched = QueueScheduler(name='test-executor-lost', retries=3) mocker.spy(sched, 'on_update') with Running(sched, name='test-scheduler'): sched.submit(task) sched.wait() assert sched.on_update.call_count == 3 states = ['TASK_STARTING', 'TASK_STARTING', 'TASK_FAILED'] for ((args, kwargs), state) in zip(sched.on_update.call_args_list, states): assert args[1].state == state
def test_runner_context_manager(): sched = QueueScheduler(name='test-scheduler') with Running(sched, name='test-scheduler'): sched.wait() assert sched
def test_docker_python_result(mocker, docker_python): sched = QueueScheduler() with Running(sched, name='test-scheduler'): sched.submit(docker_python) sched.wait() # block until all tasks finishes assert docker_python.status.data == 10
def test_docker_python_result(mocker, docker_python): sched = QueueScheduler() with Running(sched, name='test-scheduler'): result = sched.submit(docker_python) sched.wait() # block until all tasks finishes assert result.get() == 10
def test_runner_context_manager(): sched = QueueScheduler(name='test-scheduler') with Running(sched, name='test-scheduler'): sched.wait() assert sched