def test_builder_never_starts(self): test_component = yield From(self._setup_job_for_managers()) # Ensure that that the building callbacks have been registered callback_keys = [key for key in self.manager._orchestrator.callbacks] self.assertIn(JOB_PREFIX, callback_keys) # Send a signal to the callback that a worker has expired yield From( self.manager._job_callback( KeyChange( KeyEvent.EXPIRE, self.mock_job_key, json.dumps({ "had_heartbeat": False, "job_queue_item": self.mock_job.job_item }), ))) self.test_executor.stop_builder.assert_called_once_with("123") self.assertEqual(self.test_executor.stop_builder.call_count, 1) # Ensure the job was marked as incomplete, with an update_phase to True (so the DB record and # logs are updated as well) yield From( self.job_complete_callback.assert_called_once_with( ANY, BuildJobResult.INCOMPLETE, "MockExecutor", update_phase=True))
def test_another_manager_takes_job(self): # Prepare a job to be taken by another manager test_component = await self._setup_job_for_managers() await ( self.manager._realm_callback( KeyChange( KeyEvent.DELETE, slash_join(REALM_PREFIX, REALM_ID), json.dumps( { "realm": REALM_ID, "token": "beef", "execution_id": "123", "job_queue_item": self.mock_job.job_item, } ), ) ) ) self.unregister_component_callback.assert_called_once_with(test_component) # Ensure that the executor does not kill the job. self.assertEqual(self.test_executor.stop_builder.call_count, 0) # Ensure that we still have the build info, but not the component. self.assertEqual(0, self.manager.num_workers()) self.assertIsNotNone(self.manager._build_uuid_to_info.get(BUILD_UUID)) # Delete the job once it has "completed". await ( self.manager._job_callback( KeyChange( KeyEvent.DELETE, self.mock_job_key, json.dumps({"had_heartbeat": False, "job_queue_item": self.mock_job.job_item}), ) ) ) # Ensure the job was removed from the info, but stop was not called. self.assertIsNone(self.manager._build_uuid_to_info.get(BUILD_UUID)) self.assertEqual(self.test_executor.stop_builder.call_count, 0)
def test_on_key_change(orchestrator): key_prefix = "building/" mock_callback = Mock() orchestrator.on_key_change(key_prefix, lambda x: mock_callback.meth(x)) # CREATE orchestrator.set_key(slash_join(key_prefix, "key1"), "test_val") time.sleep(0.1) mock_callback.meth.assert_called_with( KeyChange( KeyEvent.CREATE, slash_join(key_prefix, "key1"), "test_val", )) # SET orchestrator.set_key(slash_join(key_prefix, "key1"), "test_val", overwrite=True) time.sleep(0.1) mock_callback.meth.assert_called_with( KeyChange( KeyEvent.SET, slash_join(key_prefix, "key1"), "test_val", )) # DELETE orchestrator.delete_key(slash_join(key_prefix, "key1")) time.sleep(0.1) mock_callback.meth.assert_called_with( KeyChange( KeyEvent.DELETE, slash_join(key_prefix, "key1"), "test_val", ))
def test_expiring_worker_not_started(self): # Ensure that that the building callbacks have been registered callback_keys = [key for key in self.manager._orchestrator.callbacks] self.assertIn(JOB_PREFIX, callback_keys) # Send a signal to the callback that a worker has expired yield From( self.manager._job_callback( KeyChange( KeyEvent.EXPIRE, self.mock_job_key, json.dumps({ 'had_heartbeat': True, 'job_queue_item': self.mock_job.job_item })))) # Since the realm was never registered, expiration should do nothing. self.assertEqual(self.test_executor.stop_builder.call_count, 0)
def test_job_started_by_other_manager(self): # Ensure that that the building callbacks have been registered callback_keys = [key for key in self.manager._orchestrator.callbacks] self.assertIn(JOB_PREFIX, callback_keys) # Send a signal to the callback that the job has been created. yield From( self.manager._job_callback( KeyChange( KeyEvent.CREATE, self.mock_job_key, json.dumps({ 'had_heartbeat': False, 'job_queue_item': self.mock_job.job_item })))) # Ensure the create does nothing. self.assertEqual(self.test_executor.stop_builder.call_count, 0)
def test_expiring_worker_started(self): test_component = yield From(self._setup_job_for_managers()) # Ensure that that the building callbacks have been registered callback_keys = [key for key in self.manager._orchestrator.callbacks] self.assertIn(JOB_PREFIX, callback_keys) yield From( self.manager._job_callback( KeyChange( KeyEvent.EXPIRE, self.mock_job_key, json.dumps({ 'had_heartbeat': True, 'job_queue_item': self.mock_job.job_item })))) self.test_executor.stop_builder.assert_called_once_with('123') self.assertEqual(self.test_executor.stop_builder.call_count, 1)
async def _setup_job_for_managers(self): test_component = Mock(spec=BuildComponent) test_component.builder_realm = REALM_ID test_component.start_build = Mock(side_effect=self._create_completed_future()) self.register_component_callback.return_value = test_component is_scheduled = await self.manager.schedule(self.mock_job) self.assertTrue(is_scheduled) self.assertEqual(self.test_executor.start_builder.call_count, 1) # Ensure that that the job, realm, and metric callbacks have been registered callback_keys = [key for key in self.manager._orchestrator.callbacks] self.assertIn(self.mock_job_key, self.manager._orchestrator.state) self.assertIn(REALM_PREFIX, callback_keys) # TODO: assert metric key has been set realm_for_build = self._find_realm_key(self.manager._orchestrator, BUILD_UUID) raw_realm_data = await ( self.manager._orchestrator.get_key(slash_join("realm", realm_for_build)) ) realm_data = json.loads(raw_realm_data) realm_data["realm"] = REALM_ID # Right now the job is not registered with any managers because etcd has not accepted the job self.assertEqual(self.register_component_callback.call_count, 0) # Fire off a realm changed with the same data. await ( self.manager._realm_callback( KeyChange( KeyEvent.CREATE, slash_join(REALM_PREFIX, REALM_ID), json.dumps(realm_data) ) ) ) # Ensure that we have at least one component node. self.assertEqual(self.register_component_callback.call_count, 1) self.assertEqual(1, self.manager.num_workers()) # Ensure that the build info exists. self.assertIsNotNone(self.manager._build_uuid_to_info.get(BUILD_UUID)) return test_component
def test_realm_expired(self): test_component = yield From(self._setup_job_for_managers()) # Send a signal to the callback that a realm has expired yield From( self.manager._realm_callback( KeyChange( KeyEvent.EXPIRE, self.mock_job_key, json.dumps({ 'realm': REALM_ID, 'execution_id': 'foobar', 'executor_name': 'MockExecutor', 'job_queue_item': { 'body': '{"build_uuid": "fakeid"}' }, })))) # Ensure that the cleanup code for the executor was called. self.test_executor.stop_builder.assert_called_once_with('foobar') self.assertEqual(self.test_executor.stop_builder.call_count, 1)
def test_buildjob_deleted(self): test_component = yield From(self._setup_job_for_managers()) # Ensure that that the building callbacks have been registered callback_keys = [key for key in self.manager._orchestrator.callbacks] self.assertIn(JOB_PREFIX, callback_keys) # Send a signal to the callback that a worker has expired yield From( self.manager._job_callback( KeyChange( KeyEvent.DELETE, self.mock_job_key, json.dumps({ 'had_heartbeat': False, 'job_queue_item': self.mock_job.job_item })))) self.assertEqual(self.test_executor.stop_builder.call_count, 0) self.assertEqual(self.job_complete_callback.call_count, 0) self.assertIsNone(self.manager._build_uuid_to_info.get(BUILD_UUID))
def test_realm_expired(self): test_component = await self._setup_job_for_managers() # Send a signal to the callback that a realm has expired await ( self.manager._realm_callback( KeyChange( KeyEvent.EXPIRE, self.mock_job_key, json.dumps( { "realm": REALM_ID, "execution_id": "foobar", "executor_name": "MockExecutor", "job_queue_item": {"body": '{"build_uuid": "fakeid"}'}, } ), ) ) ) # Ensure that the cleanup code for the executor was called. self.test_executor.stop_builder.assert_called_once_with("foobar") self.assertEqual(self.test_executor.stop_builder.call_count, 1)
def test_change_worker(self): # Send a signal to the callback that a worker key has been changed self.manager._job_callback( KeyChange(KeyEvent.SET, self.mock_job_key, "value")) self.assertEqual(self.test_executor.stop_builder.call_count, 0)