def test_single_job_dependencies_project_launch(controlplane_instance_group,
job_template_factory, mocker):
objects = job_template_factory('jt',
organization='org1',
project='proj',
inventory='inv',
credential='cred')
instance = controlplane_instance_group.instances.all()[0]
j = create_job(objects.job_template, dependencies_processed=False)
p = objects.project
p.scm_update_on_launch = True
p.scm_update_cache_timeout = 0
p.scm_type = "git"
p.scm_url = "http://github.com/ansible/ansible.git"
p.save(skip_update=True)
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
dm = DependencyManager()
with mock.patch.object(DependencyManager,
"create_project_update",
wraps=dm.create_project_update) as mock_pu:
dm.schedule()
mock_pu.assert_called_once_with(j)
pu = [x for x in p.project_updates.all()]
assert len(pu) == 1
TaskManager().schedule()
TaskManager.start_task.assert_called_once_with(
pu[0], controlplane_instance_group, [j], instance)
pu[0].status = "successful"
pu[0].save()
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
TaskManager().schedule()
TaskManager.start_task.assert_called_once_with(
j, controlplane_instance_group, [], instance)
def test_job_dependency_with_already_updated(controlplane_instance_group,
job_template_factory, mocker,
inventory_source_factory):
objects = job_template_factory('jt',
organization='org1',
project='proj',
inventory='inv',
credential='cred')
instance = controlplane_instance_group.instances.all()[0]
j = create_job(objects.job_template, dependencies_processed=False)
i = objects.inventory
ii = inventory_source_factory("ec2")
ii.source = "ec2"
ii.update_on_launch = True
ii.update_cache_timeout = 0
ii.save()
i.inventory_sources.add(ii)
j.start_args = json.dumps(dict(inventory_sources_already_updated=[ii.id]))
j.save()
j.start_args = encrypt_field(j, field_name="start_args")
j.save()
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
dm = DependencyManager()
with mock.patch.object(DependencyManager,
"create_inventory_update",
wraps=dm.create_inventory_update) as mock_iu:
dm.schedule()
mock_iu.assert_not_called()
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
TaskManager().schedule()
TaskManager.start_task.assert_called_once_with(
j, controlplane_instance_group, [], instance)
def test_single_job_dependencies_inventory_update_launch(
controlplane_instance_group, job_template_factory, mocker,
inventory_source_factory):
objects = job_template_factory('jt',
organization='org1',
project='proj',
inventory='inv',
credential='cred')
instance = controlplane_instance_group.instances.all()[0]
j = create_job(objects.job_template, dependencies_processed=False)
i = objects.inventory
ii = inventory_source_factory("ec2")
ii.source = "ec2"
ii.update_on_launch = True
ii.update_cache_timeout = 0
ii.save()
i.inventory_sources.add(ii)
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
dm = DependencyManager()
with mock.patch.object(DependencyManager,
"create_inventory_update",
wraps=dm.create_inventory_update) as mock_iu:
dm.schedule()
mock_iu.assert_called_once_with(j, ii)
iu = [x for x in ii.inventory_updates.all()]
assert len(iu) == 1
TaskManager().schedule()
TaskManager.start_task.assert_called_once_with(
iu[0], controlplane_instance_group, [j], instance)
iu[0].status = "successful"
iu[0].save()
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
TaskManager().schedule()
TaskManager.start_task.assert_called_once_with(
j, controlplane_instance_group, [], instance)
def test_job_not_blocking_inventory_update(controlplane_instance_group,
job_template_factory,
inventory_source_factory):
instance = controlplane_instance_group.instances.all()[0]
objects = job_template_factory('jt',
organization='org1',
project='proj',
inventory='inv',
credential='cred',
jobs=["job"])
job = objects.jobs["job"]
job.instance_group = controlplane_instance_group
job.status = "running"
job.save()
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
inv = objects.inventory
inv_source = inventory_source_factory("ec2")
inv_source.source = "ec2"
inv.inventory_sources.add(inv_source)
inventory_update = inv_source.create_inventory_update()
inventory_update.instance_group = controlplane_instance_group
inventory_update.status = "pending"
inventory_update.save()
DependencyManager().schedule()
TaskManager().schedule()
TaskManager.start_task.assert_called_once_with(
inventory_update, controlplane_instance_group, [], instance)
def test_inventory_update_launches_project_update(controlplane_instance_group,
scm_inventory_source):
ii = scm_inventory_source
project = scm_inventory_source.source_project
project.scm_update_on_launch = True
project.save()
iu = ii.create_inventory_update()
iu.status = "pending"
iu.save()
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
dm = DependencyManager()
with mock.patch.object(DependencyManager,
"create_project_update",
wraps=dm.create_project_update) as mock_pu:
dm.schedule()
mock_pu.assert_called_with(iu, project_id=project.id)
def test_control_and_execution_instance(self, project, system_job_template,
job_template, inventory_source,
control_instance,
execution_instance):
assert Instance.objects.count() == 2
pu = project.create_unified_job()
sj = system_job_template.create_unified_job()
job = job_template.create_unified_job()
inv_update = inventory_source.create_unified_job()
all_ujs = (pu, sj, job, inv_update)
for uj in all_ujs:
uj.signal_start()
DependencyManager().schedule()
tm = TaskManager()
self.run_tm(tm)
for uj in all_ujs:
uj.refresh_from_db()
assert uj.status == 'waiting'
for uj in (pu, sj): # control plane jobs
assert uj.capacity_type == 'control'
assert [uj.execution_node, uj.controller_node] == [
control_instance.hostname, control_instance.hostname
], uj
for uj in (job, inv_update): # user-space jobs
assert uj.capacity_type == 'execution'
assert [uj.execution_node, uj.controller_node] == [
execution_instance.hostname, control_instance.hostname
], uj
def get(self, request):
DependencyManager().schedule()
if not settings.AWX_DISABLE_TASK_MANAGERS:
msg = f"Running {self.prefix} manager. To disable other triggers to the {self.prefix} manager, set AWX_DISABLE_TASK_MANAGERS to True"
else:
msg = f"AWX_DISABLE_TASK_MANAGERS is True, this view is the only way to trigger the {self.prefix} manager"
return Response(msg)
def test_multi_group_with_shared_dependency(instance_factory,
controlplane_instance_group,
mocker, instance_group_factory,
job_template_factory):
i1 = instance_factory("i1")
i2 = instance_factory("i2")
ig1 = instance_group_factory("ig1", instances=[i1])
ig2 = instance_group_factory("ig2", instances=[i2])
objects1 = job_template_factory(
'jt1',
organization='org1',
project='proj1',
inventory='inv1',
credential='cred1',
)
objects1.job_template.instance_groups.add(ig1)
j1 = create_job(objects1.job_template, dependencies_processed=False)
p = objects1.project
p.scm_update_on_launch = True
p.scm_update_cache_timeout = 0
p.scm_type = "git"
p.scm_url = "http://github.com/ansible/ansible.git"
p.save()
objects2 = job_template_factory('jt2',
organization=objects1.organization,
project=p,
inventory='inv2',
credential='cred2')
objects2.job_template.instance_groups.add(ig2)
j2 = create_job(objects2.job_template, dependencies_processed=False)
with mocker.patch("awx.main.scheduler.TaskManager.start_task"):
DependencyManager().schedule()
TaskManager().schedule()
pu = p.project_updates.first()
TaskManager.start_task.assert_called_once_with(
pu, controlplane_instance_group, [j1, j2],
controlplane_instance_group.instances.all()[0])
pu.finished = pu.created + timedelta(seconds=1)
pu.status = "successful"
pu.save()
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
DependencyManager().schedule()
TaskManager().schedule()
TaskManager.start_task.assert_any_call(j1, ig1, [], i1)
TaskManager.start_task.assert_any_call(j2, ig2, [], i2)
assert TaskManager.start_task.call_count == 2
def test_shared_dependencies_launch(controlplane_instance_group,
job_template_factory, mocker,
inventory_source_factory):
instance = controlplane_instance_group.instances.all()[0]
objects = job_template_factory('jt',
organization='org1',
project='proj',
inventory='inv',
credential='cred')
objects.job_template.allow_simultaneous = True
objects.job_template.save()
j1 = create_job(objects.job_template, dependencies_processed=False)
j2 = create_job(objects.job_template, dependencies_processed=False)
p = objects.project
p.scm_update_on_launch = True
p.scm_update_cache_timeout = 300
p.scm_type = "git"
p.scm_url = "http://github.com/ansible/ansible.git"
p.save()
i = objects.inventory
ii = inventory_source_factory("ec2")
ii.source = "ec2"
ii.update_on_launch = True
ii.update_cache_timeout = 300
ii.save()
i.inventory_sources.add(ii)
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
DependencyManager().schedule()
TaskManager().schedule()
pu = p.project_updates.first()
iu = ii.inventory_updates.first()
TaskManager.start_task.assert_has_calls([
mock.call(iu, controlplane_instance_group, [j1, j2], instance),
mock.call(pu, controlplane_instance_group, [j1, j2], instance)
])
pu.status = "successful"
pu.finished = pu.created + timedelta(seconds=1)
pu.save()
iu.status = "successful"
iu.finished = iu.created + timedelta(seconds=1)
iu.save()
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
TaskManager().schedule()
TaskManager.start_task.assert_has_calls([
mock.call(j1, controlplane_instance_group, [], instance),
mock.call(j2, controlplane_instance_group, [], instance)
])
pu = [x for x in p.project_updates.all()]
iu = [x for x in ii.inventory_updates.all()]
assert len(pu) == 1
assert len(iu) == 1
def test_approval_node_approve(self, post, admin_user, job_template,
controlplane_instance_group):
# This test ensures that a user (with permissions to do so) can APPROVE
# workflow approvals. Also asserts that trying to APPROVE approvals
# that have already been dealt with will throw an error.
wfjt = WorkflowJobTemplate.objects.create(name='foobar')
node = wfjt.workflow_nodes.create(unified_job_template=job_template)
url = reverse('api:workflow_job_template_node_create_approval',
kwargs={
'pk': node.pk,
'version': 'v2'
})
post(url, {
'name': 'Approve Test',
'description': '',
'timeout': 0
},
user=admin_user,
expect=201)
post(reverse('api:workflow_job_template_launch',
kwargs={'pk': wfjt.pk}),
user=admin_user,
expect=201)
wf_job = WorkflowJob.objects.first()
DependencyManager().schedule(
) # TODO: exclude workflows from this and delete line
TaskManager().schedule()
WorkflowManager().schedule()
wfj_node = wf_job.workflow_nodes.first()
approval = wfj_node.job
assert approval.name == 'Approve Test'
post(reverse('api:workflow_approval_approve',
kwargs={'pk': approval.pk}),
user=admin_user,
expect=204)
# Test that there is an activity stream entry that was created for the "approve" action.
qs = ActivityStream.objects.order_by('-timestamp').first()
assert qs.object1 == 'workflow_approval'
assert qs.changes == '{"status": ["pending", "successful"]}'
assert WorkflowApproval.objects.get(
pk=approval.pk).status == 'successful'
assert qs.operation == 'update'
post(reverse('api:workflow_approval_approve',
kwargs={'pk': approval.pk}),
user=admin_user,
expect=400)
def test_task_manager_workflow_rescheduling(self, job_template_factory,
inventory, project,
controlplane_instance_group):
jt = JobTemplate.objects.create(allow_simultaneous=True,
inventory=inventory,
project=project,
playbook='helloworld.yml')
wfjt = WorkflowJobTemplate.objects.create(name='foo')
for i in range(2):
wfjt.workflow_nodes.create(unified_job_template=jt)
wj = wfjt.create_unified_job()
assert wj.workflow_nodes.count() == 2
wj.signal_start()
# Transitions workflow job to running
# needs to re-schedule so it spawns jobs next round
self.run_tm(TaskManager(), [mock.call('running')])
# Spawns jobs
# needs re-schedule to submit jobs next round
self.run_tm(
WorkflowManager(),
[mock.call('pending'), mock.call('pending')])
assert jt.jobs.count() == 2 # task manager spawned jobs
# Submits jobs
# intermission - jobs will run and reschedule TM when finished
self.run_tm(DependencyManager()) # flip dependencies_processed to True
self.run_tm(
TaskManager(),
[mock.call('waiting'), mock.call('waiting')])
# I am the job runner
for job in jt.jobs.all():
job.status = 'successful'
job.save()
# Finishes workflow
# no further action is necessary, so rescheduling should not happen
self.run_tm(WorkflowManager(), [mock.call('successful')])
def test_hybrid_capacity(self, job_template, hybrid_instance):
enough_capacity = job_template.create_unified_job()
insufficient_capacity = job_template.create_unified_job()
expected_task_impact = enough_capacity.task_impact + settings.AWX_CONTROL_NODE_TASK_IMPACT
all_ujs = [enough_capacity, insufficient_capacity]
for uj in all_ujs:
uj.signal_start()
DependencyManager().schedule()
# There is only enough control capacity to run one of the jobs so one should end up in pending and the other in waiting
tm = TaskManager()
self.run_tm(tm)
for uj in all_ujs:
uj.refresh_from_db()
assert enough_capacity.status == 'waiting'
assert insufficient_capacity.status == 'pending'
assert [
enough_capacity.execution_node, enough_capacity.controller_node
] == [
hybrid_instance.hostname,
hybrid_instance.hostname,
], enough_capacity
assert expected_task_impact == hybrid_instance.consumed_capacity
def test_job_fails_to_launch_when_no_control_capacity(
self, job_template, control_instance_low_capacity,
execution_instance):
enough_capacity = job_template.create_unified_job()
insufficient_capacity = job_template.create_unified_job()
all_ujs = [enough_capacity, insufficient_capacity]
for uj in all_ujs:
uj.signal_start()
DependencyManager().schedule()
# There is only enough control capacity to run one of the jobs so one should end up in pending and the other in waiting
tm = TaskManager()
self.run_tm(tm)
for uj in all_ujs:
uj.refresh_from_db()
assert enough_capacity.status == 'waiting'
assert insufficient_capacity.status == 'pending'
assert [
enough_capacity.execution_node, enough_capacity.controller_node
] == [
execution_instance.hostname,
control_instance_low_capacity.hostname,
], enough_capacity
def test_generate_dependencies_only_once(job_template_factory):
objects = job_template_factory('jt', organization='org1')
job = objects.job_template.create_job()
job.status = "pending"
job.name = "job_gen_dep"
job.save()
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
# job starts with dependencies_processed as False
assert not job.dependencies_processed
# run one cycle of ._schedule() to generate dependencies
DependencyManager().schedule()
# make sure dependencies_processed is now True
job = Job.objects.filter(name="job_gen_dep")[0]
assert job.dependencies_processed
# Run ._schedule() again, but make sure .generate_dependencies() is not
# called with job in the argument list
dm = DependencyManager()
dm.generate_dependencies = mock.MagicMock(return_value=[])
dm.schedule()
dm.generate_dependencies.assert_not_called()