def test_timeout_task(dbsession, engine):
task1 = Task(name='task1', active=True, retries=2)
dbsession.add(task1)
taskflow = Taskflow()
taskflow.add_task(task1)
task_instance = task1.get_new_instance(run_at=datetime(2017, 6, 4, 6))
dbsession.add(task_instance)
dbsession.commit()
task_instance_id = task_instance.id
dbsession.expunge_all()
pulled_task_instances = taskflow.pull(dbsession, 'test', now=datetime(2017, 6, 4, 6, 0, 12))
pulled_task_instance = pulled_task_instances[0]
assert pulled_task_instance.id == task_instance_id
assert pulled_task_instance.status == 'running'
assert pulled_task_instance.locked_at == datetime(2017, 6, 4, 6, 0, 12)
assert pulled_task_instance.started_at == datetime(2017, 6, 4, 6, 0, 12)
assert pulled_task_instance.worker_id == 'test'
dbsession.expunge_all()
## pull 5 minutes later
pulled_task_instances = taskflow.pull(dbsession, 'test2', now=datetime(2017, 6, 4, 6, 5, 15))
pulled_task_instance = pulled_task_instances[0]
assert pulled_task_instance.status == 'running'
assert pulled_task_instance.locked_at == datetime(2017, 6, 4, 6, 5, 15)
assert pulled_task_instance.worker_id == 'test2'
assert pulled_task_instance.started_at == datetime(2017, 6, 4, 6, 0, 12)
assert pulled_task_instance.attempts == 2
def test_fail_task(dbsession, engine):
task1 = Task(name='task1', active=True)
dbsession.add(task1)
taskflow = Taskflow()
taskflow.add_task(task1)
task_instance = task1.get_new_instance(run_at=datetime(2017, 6, 4, 6))
dbsession.add(task_instance)
dbsession.commit()
task_instance_id = task_instance.id
dbsession.expunge_all()
pulled_task_instances = taskflow.pull(dbsession, 'test', now=datetime(2017, 6, 4, 6, 0, 12))
pulled_task_instance = pulled_task_instances[0]
assert pulled_task_instance.id == task_instance_id
assert pulled_task_instance.status == 'running'
assert pulled_task_instance.locked_at == datetime(2017, 6, 4, 6, 0, 12)
assert pulled_task_instance.started_at == datetime(2017, 6, 4, 6, 0, 12)
assert pulled_task_instance.worker_id == 'test'
pulled_task_instance.fail(dbsession, taskflow, now=datetime(2017, 6, 4, 6, 0, 15))
dbsession.refresh(pulled_task_instance)
assert pulled_task_instance.status == 'failed'
assert pulled_task_instance.ended_at == datetime(2017, 6, 4, 6, 0, 15)
def test_queue_pull_task_run_at_order(dbsession, engine):
task1 = Task(name='task1', active=True)
dbsession.add(task1)
taskflow = Taskflow()
taskflow.add_task(task1)
task_instance1 = task1.get_new_instance(run_at=datetime(2017, 6, 4, 6, 0, 5))
task_instance2 = task1.get_new_instance(run_at=datetime(2017, 6, 4, 6, 0, 10))
dbsession.add(task_instance1)
dbsession.add(task_instance2)
dbsession.commit()
sooner_task_instance_id = task_instance1.id
later_task_instance_id = task_instance2.id
dbsession.expunge_all()
pulled_task_instances = taskflow.pull(dbsession, 'test', max_tasks=1, now=datetime(2017, 6, 4, 6, 0, 12))
pulled_task_instance = pulled_task_instances[0]
assert pulled_task_instance.id == sooner_task_instance_id
assert pulled_task_instance.status == 'running'
assert pulled_task_instance.locked_at == datetime(2017, 6, 4, 6, 0, 12)
assert pulled_task_instance.started_at == datetime(2017, 6, 4, 6, 0, 12)
assert pulled_task_instance.worker_id == 'test'
pulled_task_instances = taskflow.pull(dbsession, 'test', max_tasks=1, now=datetime(2017, 6, 4, 6, 0, 12))
pulled_task_instance = pulled_task_instances[0]
assert pulled_task_instance.id == later_task_instance_id
assert pulled_task_instance.status == 'running'
assert pulled_task_instance.locked_at == datetime(2017, 6, 4, 6, 0, 12)
assert pulled_task_instance.started_at == datetime(2017, 6, 4, 6, 0, 12)
assert pulled_task_instance.worker_id == 'test'
def tasks(dbsession):
task1 = Task(name='task1', active=True)
task2 = Task(name='task2', active=True, schedule='0 6 * * *')
task3 = Task(name='task3', active=True)
task4 = Task(name='task4', active=True, schedule='0 2 * * *')
dbsession.add(task1)
dbsession.add(task2)
dbsession.add(task3)
dbsession.add(task4)
dbsession.commit()
return [task1, task2, task3, task4]
def test_queue_unique(dbsession, engine):
task1 = Task(name='task1', active=True)
dbsession.add(task1)
taskflow = Taskflow()
taskflow.add_task(task1)
task_instance1 = task1.get_new_instance(run_at=datetime(2017, 6, 4, 6), unique='foo')
dbsession.add(task_instance1)
dbsession.commit()
with pytest.raises(IntegrityError):
task_instance2 = task1.get_new_instance(run_at=datetime(2017, 6, 4, 8), unique='foo')
dbsession.add(task_instance2)
dbsession.commit()
def test_task_retry_success(dbsession, engine):
task1 = Task(name='task1', active=True, retries=1)
dbsession.add(task1)
taskflow = Taskflow()
taskflow.add_task(task1)
task_instance = task1.get_new_instance(run_at=datetime(2017, 6, 4, 6))
dbsession.add(task_instance)
dbsession.commit()
task_instance_id = task_instance.id
dbsession.expunge_all()
now = datetime(2017, 6, 4, 6, 0, 12)
pulled_task_instances = taskflow.pull(dbsession, 'test', now=now)
pulled_task_instance = pulled_task_instances[0]
assert pulled_task_instance.id == task_instance_id
assert pulled_task_instance.status == 'running'
assert pulled_task_instance.locked_at == now
assert pulled_task_instance.started_at == now
assert pulled_task_instance.worker_id == 'test'
pulled_task_instance.fail(dbsession, taskflow, now=datetime(2017, 6, 4, 6, 0, 15))
dbsession.refresh(pulled_task_instance)
assert pulled_task_instance.status == 'retry'
dbsession.expunge_all()
## stays within retry_delay
pulled_task_instances = taskflow.pull(dbsession, 'test', now=datetime(2017, 6, 4, 6, 5, 14))
assert pulled_task_instances == []
pulled_task_instances = taskflow.pull(dbsession, 'test2', now=datetime(2017, 6, 4, 6, 5, 16))
pulled_task_instance = pulled_task_instances[0]
assert pulled_task_instance.status == 'running'
assert pulled_task_instance.locked_at == datetime(2017, 6, 4, 6, 5, 16)
assert pulled_task_instance.started_at == datetime(2017, 6, 4, 6, 0, 12)
assert pulled_task_instance.attempts == 2
assert pulled_task_instance.worker_id == 'test2'
pulled_task_instance.succeed(dbsession, taskflow, now=datetime(2017, 6, 4, 6, 5, 20))
dbsession.refresh(pulled_task_instance)
assert pulled_task_instance.status == 'success'
assert pulled_task_instance.ended_at == datetime(2017, 6, 4, 6, 5, 20)
def test_fail_retry(dbsession, monkeypatch):
mock_aws_batch = MockAWSBatch([], 'SUBMITTED')
monkeypatch.setattr(boto3, 'client', mockbatch(mock_aws_batch))
task1 = Task(name='task1',
active=True,
push_destination='aws_batch',
retries=1)
dbsession.add(task1)
taskflow = Taskflow()
taskflow.add_task(task1)
taskflow.add_push_worker(AWSBatchPushWorker(taskflow))
taskflow.sync_db(dbsession)
task_instance = task1.get_new_instance(run_at=datetime(2017, 6, 4, 6))
dbsession.add(task_instance)
dbsession.commit()
pusher = Pusher(taskflow, now_override=datetime(2017, 6, 4, 6))
pusher.run(dbsession)
task_instances = dbsession.query(TaskInstance).all()
pushed_task_instance = task_instances[0]
assert pushed_task_instance.status == 'pushed'
assert 'jobId' in pushed_task_instance.push_state
assert pushed_task_instance.push_state['jobName'] == 'task1__1'
mock_aws_batch.status = 'FAILED'
pusher.run(dbsession)
assert pushed_task_instance.status == 'retry'
assert pushed_task_instance.ended_at == None
mock_aws_batch.remove_job(pushed_task_instance.push_state['jobId'])
pusher.now_override = datetime(2017, 6, 4, 6, 5)
mock_aws_batch.status = 'FAILED'
pusher.run(dbsession)
assert pushed_task_instance.status == 'retry'
assert pushed_task_instance.ended_at == None
mock_aws_batch.remove_job(pushed_task_instance.push_state['jobId'])
pusher.now_override = datetime(2017, 6, 4, 6, 10)
mock_aws_batch.status = 'FAILED'
pusher.run(dbsession)
assert pushed_task_instance.status == 'failed'
assert pushed_task_instance.ended_at == datetime(2017, 6, 4, 6, 10)
def workflows(dbsession):
workflow1 = Workflow(name='workflow1', active=True, schedule='0 6 * * *')
workflow2 = Workflow(name='workflow2', active=True)
dbsession.add(workflow1)
dbsession.add(workflow2)
dbsession.commit()
task1 = Task(workflow=workflow1, name='task1', active=True)
task2 = Task(workflow=workflow1, name='task2', active=True)
task3 = Task(workflow=workflow1, name='task3', active=True)
task4 = Task(workflow=workflow1, name='task4', active=True)
task3.depends_on(task1)
task3.depends_on(task2)
task4.depends_on(task3)
dbsession.add(task1)
dbsession.add(task2)
dbsession.add(task3)
dbsession.add(task4)
dbsession.commit()
return [workflow1, workflow2]
def test_queue_pull_task_priority(dbsession, engine):
task1 = Task(name='task1', active=True)
dbsession.add(task1)
taskflow = Taskflow()
taskflow.add_task(task1)
task_instance1 = task1.get_new_instance(run_at=datetime(2017, 6, 4, 6))
task_instance2 = task1.get_new_instance(run_at=datetime(2017, 6, 4, 6), priority='high')
dbsession.add(task_instance1)
dbsession.add(task_instance2)
dbsession.commit()
normal_task_instance_id = task_instance1.id
high_task_instance_id = task_instance2.id
dbsession.expunge_all()
pulled_task_instances = taskflow.pull(dbsession, 'test', max_tasks=1, now=datetime(2017, 6, 4, 6, 0, 12))
pulled_task_instance = pulled_task_instances[0]
assert pulled_task_instance.id == high_task_instance_id
assert pulled_task_instance.status == 'running'
assert pulled_task_instance.locked_at == datetime(2017, 6, 4, 6, 0, 12)
assert pulled_task_instance.started_at == datetime(2017, 6, 4, 6, 0, 12)
assert pulled_task_instance.worker_id == 'test'
assert pulled_task_instance.priority == 'high'
pulled_task_instances = taskflow.pull(dbsession, 'test', max_tasks=1, now=datetime(2017, 6, 4, 6, 0, 12))
pulled_task_instance = pulled_task_instances[0]
assert pulled_task_instance.id == normal_task_instance_id
assert pulled_task_instance.status == 'running'
assert pulled_task_instance.locked_at == datetime(2017, 6, 4, 6, 0, 12)
assert pulled_task_instance.started_at == datetime(2017, 6, 4, 6, 0, 12)
assert pulled_task_instance.worker_id == 'test'
assert pulled_task_instance.priority == 'normal'