def test_builder_never_starts(self):
test_component = yield From(self._setup_job_for_managers())
# Ensure that that the building callbacks have been registered
callback_keys = [key for key in self.manager._orchestrator.callbacks]
self.assertIn(JOB_PREFIX, callback_keys)
# Send a signal to the callback that a worker has expired
yield From(
self.manager._job_callback(
KeyChange(
KeyEvent.EXPIRE,
self.mock_job_key,
json.dumps({
"had_heartbeat": False,
"job_queue_item": self.mock_job.job_item
}),
)))
self.test_executor.stop_builder.assert_called_once_with("123")
self.assertEqual(self.test_executor.stop_builder.call_count, 1)
# Ensure the job was marked as incomplete, with an update_phase to True (so the DB record and
# logs are updated as well)
yield From(
self.job_complete_callback.assert_called_once_with(
ANY,
BuildJobResult.INCOMPLETE,
"MockExecutor",
update_phase=True))
def test_another_manager_takes_job(self):
# Prepare a job to be taken by another manager
test_component = await self._setup_job_for_managers()
await (
self.manager._realm_callback(
KeyChange(
KeyEvent.DELETE,
slash_join(REALM_PREFIX, REALM_ID),
json.dumps(
{
"realm": REALM_ID,
"token": "beef",
"execution_id": "123",
"job_queue_item": self.mock_job.job_item,
}
),
)
)
)
self.unregister_component_callback.assert_called_once_with(test_component)
# Ensure that the executor does not kill the job.
self.assertEqual(self.test_executor.stop_builder.call_count, 0)
# Ensure that we still have the build info, but not the component.
self.assertEqual(0, self.manager.num_workers())
self.assertIsNotNone(self.manager._build_uuid_to_info.get(BUILD_UUID))
# Delete the job once it has "completed".
await (
self.manager._job_callback(
KeyChange(
KeyEvent.DELETE,
self.mock_job_key,
json.dumps({"had_heartbeat": False, "job_queue_item": self.mock_job.job_item}),
)
)
)
# Ensure the job was removed from the info, but stop was not called.
self.assertIsNone(self.manager._build_uuid_to_info.get(BUILD_UUID))
self.assertEqual(self.test_executor.stop_builder.call_count, 0)
def test_on_key_change(orchestrator):
key_prefix = "building/"
mock_callback = Mock()
orchestrator.on_key_change(key_prefix, lambda x: mock_callback.meth(x))
# CREATE
orchestrator.set_key(slash_join(key_prefix, "key1"), "test_val")
time.sleep(0.1)
mock_callback.meth.assert_called_with(
KeyChange(
KeyEvent.CREATE,
slash_join(key_prefix, "key1"),
"test_val",
))
# SET
orchestrator.set_key(slash_join(key_prefix, "key1"),
"test_val",
overwrite=True)
time.sleep(0.1)
mock_callback.meth.assert_called_with(
KeyChange(
KeyEvent.SET,
slash_join(key_prefix, "key1"),
"test_val",
))
# DELETE
orchestrator.delete_key(slash_join(key_prefix, "key1"))
time.sleep(0.1)
mock_callback.meth.assert_called_with(
KeyChange(
KeyEvent.DELETE,
slash_join(key_prefix, "key1"),
"test_val",
))
def test_expiring_worker_not_started(self):
# Ensure that that the building callbacks have been registered
callback_keys = [key for key in self.manager._orchestrator.callbacks]
self.assertIn(JOB_PREFIX, callback_keys)
# Send a signal to the callback that a worker has expired
yield From(
self.manager._job_callback(
KeyChange(
KeyEvent.EXPIRE, self.mock_job_key,
json.dumps({
'had_heartbeat': True,
'job_queue_item': self.mock_job.job_item
}))))
# Since the realm was never registered, expiration should do nothing.
self.assertEqual(self.test_executor.stop_builder.call_count, 0)
def test_job_started_by_other_manager(self):
# Ensure that that the building callbacks have been registered
callback_keys = [key for key in self.manager._orchestrator.callbacks]
self.assertIn(JOB_PREFIX, callback_keys)
# Send a signal to the callback that the job has been created.
yield From(
self.manager._job_callback(
KeyChange(
KeyEvent.CREATE, self.mock_job_key,
json.dumps({
'had_heartbeat': False,
'job_queue_item': self.mock_job.job_item
}))))
# Ensure the create does nothing.
self.assertEqual(self.test_executor.stop_builder.call_count, 0)
def test_expiring_worker_started(self):
test_component = yield From(self._setup_job_for_managers())
# Ensure that that the building callbacks have been registered
callback_keys = [key for key in self.manager._orchestrator.callbacks]
self.assertIn(JOB_PREFIX, callback_keys)
yield From(
self.manager._job_callback(
KeyChange(
KeyEvent.EXPIRE, self.mock_job_key,
json.dumps({
'had_heartbeat': True,
'job_queue_item': self.mock_job.job_item
}))))
self.test_executor.stop_builder.assert_called_once_with('123')
self.assertEqual(self.test_executor.stop_builder.call_count, 1)
async def _setup_job_for_managers(self):
test_component = Mock(spec=BuildComponent)
test_component.builder_realm = REALM_ID
test_component.start_build = Mock(side_effect=self._create_completed_future())
self.register_component_callback.return_value = test_component
is_scheduled = await self.manager.schedule(self.mock_job)
self.assertTrue(is_scheduled)
self.assertEqual(self.test_executor.start_builder.call_count, 1)
# Ensure that that the job, realm, and metric callbacks have been registered
callback_keys = [key for key in self.manager._orchestrator.callbacks]
self.assertIn(self.mock_job_key, self.manager._orchestrator.state)
self.assertIn(REALM_PREFIX, callback_keys)
# TODO: assert metric key has been set
realm_for_build = self._find_realm_key(self.manager._orchestrator, BUILD_UUID)
raw_realm_data = await (
self.manager._orchestrator.get_key(slash_join("realm", realm_for_build))
)
realm_data = json.loads(raw_realm_data)
realm_data["realm"] = REALM_ID
# Right now the job is not registered with any managers because etcd has not accepted the job
self.assertEqual(self.register_component_callback.call_count, 0)
# Fire off a realm changed with the same data.
await (
self.manager._realm_callback(
KeyChange(
KeyEvent.CREATE, slash_join(REALM_PREFIX, REALM_ID), json.dumps(realm_data)
)
)
)
# Ensure that we have at least one component node.
self.assertEqual(self.register_component_callback.call_count, 1)
self.assertEqual(1, self.manager.num_workers())
# Ensure that the build info exists.
self.assertIsNotNone(self.manager._build_uuid_to_info.get(BUILD_UUID))
return test_component
def test_realm_expired(self):
test_component = yield From(self._setup_job_for_managers())
# Send a signal to the callback that a realm has expired
yield From(
self.manager._realm_callback(
KeyChange(
KeyEvent.EXPIRE, self.mock_job_key,
json.dumps({
'realm': REALM_ID,
'execution_id': 'foobar',
'executor_name': 'MockExecutor',
'job_queue_item': {
'body': '{"build_uuid": "fakeid"}'
},
}))))
# Ensure that the cleanup code for the executor was called.
self.test_executor.stop_builder.assert_called_once_with('foobar')
self.assertEqual(self.test_executor.stop_builder.call_count, 1)
def test_buildjob_deleted(self):
test_component = yield From(self._setup_job_for_managers())
# Ensure that that the building callbacks have been registered
callback_keys = [key for key in self.manager._orchestrator.callbacks]
self.assertIn(JOB_PREFIX, callback_keys)
# Send a signal to the callback that a worker has expired
yield From(
self.manager._job_callback(
KeyChange(
KeyEvent.DELETE, self.mock_job_key,
json.dumps({
'had_heartbeat': False,
'job_queue_item': self.mock_job.job_item
}))))
self.assertEqual(self.test_executor.stop_builder.call_count, 0)
self.assertEqual(self.job_complete_callback.call_count, 0)
self.assertIsNone(self.manager._build_uuid_to_info.get(BUILD_UUID))
def test_realm_expired(self):
test_component = await self._setup_job_for_managers()
# Send a signal to the callback that a realm has expired
await (
self.manager._realm_callback(
KeyChange(
KeyEvent.EXPIRE,
self.mock_job_key,
json.dumps(
{
"realm": REALM_ID,
"execution_id": "foobar",
"executor_name": "MockExecutor",
"job_queue_item": {"body": '{"build_uuid": "fakeid"}'},
}
),
)
)
)
# Ensure that the cleanup code for the executor was called.
self.test_executor.stop_builder.assert_called_once_with("foobar")
self.assertEqual(self.test_executor.stop_builder.call_count, 1)
def test_change_worker(self):
# Send a signal to the callback that a worker key has been changed
self.manager._job_callback(
KeyChange(KeyEvent.SET, self.mock_job_key, "value"))
self.assertEqual(self.test_executor.stop_builder.call_count, 0)