def setUp(self): async def mock_set_state(drone, state): drone.state = state f = asyncio.Future() f.set_result(None) return f self.drone = self.mock_drone.return_value self.drone.resource_attributes = AttributeDict( drone_uuid="test-923ABF", remote_resource_uuid="0815", updated=datetime.now() - timedelta(minutes=10), ) self.drone.demand = 8.0 self.drone._supply = 8.0 self.drone.minimum_lifetime = 3600 self.drone.set_state.side_effect = partial(mock_set_state, self.drone) self.drone.site_agent.deploy_resource.return_value = async_return( return_value=AttributeDict() ) self.drone.site_agent.resource_status.return_value = async_return( return_value=AttributeDict() ) self.drone.site_agent.stop_resource.return_value = async_return( return_value=AttributeDict() ) self.drone.site_agent.terminate_resource.return_value = async_return( return_value=AttributeDict() ) self.drone.batch_system_agent.integrate_machine.return_value = async_return( return_value=None ) self.drone.batch_system_agent.disintegrate_machine.return_value = async_return( return_value=None ) self.drone.batch_system_agent.drain_machine.return_value = async_return( return_value=None ) self.drone.batch_system_agent.get_machine_status.return_value = async_return( return_value=None ) self.drone.batch_system_agent.get_allocation.return_value = async_return( return_value=None ) self.drone.batch_system_agent.get_utilisation.return_value = async_return( return_value=None )
def test_resource_status(self): self.assertEqual( run_async( self.openstack_adapter.resource_status, resource_attributes=AttributeDict( remote_resource_uuid="029312-1231-123123"), ), AttributeDict( drone_uuid="testsite-089123", remote_resource_uuid="029312-1231-123123", resource_status=ResourceStatus.Running, ), ) self.mock_openstack_api.return_value.init_api.assert_called_with( timeout=60) self.mock_openstack_api.return_value.servers.get.assert_called_with( "029312-1231-123123")
def resource_attributes(self): return AttributeDict( machine_type="test2large", site_name="TestSite", remote_resource_uuid=1390065, resource_status=ResourceStatus.Booting, drone_uuid="testsite-1390065", )
def test_resource_status_raise_future(self): future_timestamp = datetime.now() + timedelta(minutes=1) with self.assertRaises(TardisResourceStatusUpdateFailed): with self.assertLogs(logging.getLogger(), logging.ERROR): run_async( self.adapter.resource_status, AttributeDict(remote_resource_uuid="1351043", created=future_timestamp))
def wrapper(self): executor = self.mock_executor.return_value executor.run_command.return_value = async_return( return_value=AttributeDict( stdout=stdout, stderr=stderr, exit_code=exit_code)) executor.run_command.side_effect = raise_exception func(self) executor.run_command.side_effect = None
def test_terminate_resource(self): run_async( self.cloudstack_adapter.terminate_resource, resource_attributes=AttributeDict(remote_resource_uuid="123456"), ) self.mock_cloudstack_api.return_value.destroyVirtualMachine.assert_called_with( id="123456")
def test_deploy_resource(self): self.assertEqual( run_async( self.openstack_adapter.deploy_resource, resource_attributes=AttributeDict(drone_uuid="testsite-089123"), ), AttributeDict(drone_uuid="testsite-089123"), ) self.mock_openstack_api.return_value.init_api.assert_called_with(timeout=60) self.mock_openstack_api.return_value.servers.create.assert_called_with( server={ "imageRef": "bc613271-6a54-48ca-9222-47e009dc0c29", "name": "testsite-089123", } )
def run_the_matrix(self, matrix, initial_state): for resource_status, machine_status, new_state in matrix: self.drone.site_agent.resource_status.return_value = async_return( return_value=AttributeDict(resource_status=resource_status)) self.drone.batch_system_agent.get_machine_status.return_value = async_return( return_value=machine_status) self.drone.state.return_value = initial_state run_async(self.drone.state.return_value.run, self.drone) self.assertIsInstance(self.drone.state, new_state)
def test_resource_status_raise_past(self): # Update interval is 10 minutes, so set last update back by 11 minutes in order to execute condor_q command and # creation date to 12 minutes ago past_timestamp = datetime.now() - timedelta(minutes=12) self.adapter._htcondor_queue._last_update = datetime.now() - timedelta(minutes=11) with self.assertLogs(logging.getLogger(), logging.ERROR): response = run_async(self.adapter.resource_status, AttributeDict(remote_resource_uuid="1351043", created=past_timestamp)) self.assertEqual(response.resource_status, ResourceStatus.Deleted)
def get_test_param(drone_uuid, resource_status): return AttributeDict( site_name="test-site", machine_type="test_machine_type", created=datetime.now(), updated=datetime.now(), drone_uuid=drone_uuid, resource_status=resource_status, )
def test_update_configuration(self): with open(os.path.join(self.test_path, "OpenStack.yml"), "r") as config_file: config_file_content = yaml.safe_load(config_file) self.configuration1 = Configuration(config_file_content) self.assertEqual( self.configuration1.OpenStack, AttributeDict(api_key="qwertzuiop", api_secret="katze123"), )
def mocked_get_user(user_name): if user_name == "test": return AttributeDict( user_name="test", hashed_password= "******", # noqa B509 scopes=["resources:get"], ) return None
def setUp(self) -> None: self.mock_site_agent.machine_meta_data = AttributeDict(Cores=8) self.mock_site_agent.drone_minimum_lifetime = None self.mock_plugin = MagicMock(spec=Plugin)() self.mock_plugin.notify.return_value = async_return() self.drone = Drone( site_agent=self.mock_site_agent, batch_system_agent=self.mock_batch_system_agent, )
def test_get_utilisation(self): self.assertEqual( run_async(self.fake_adapter.get_utilisation, "test-123"), 1.0) self.config.BatchSystem.utilisation = AttributeDict( get_value=lambda: 0.9) self.fake_adapter = FakeBatchSystemAdapter() self.assertEqual( run_async(self.fake_adapter.get_utilisation, "test-123"), 0.9)
def test_deploy_resource(self): response = run_async(self.adapter.deploy_resource, AttributeDict(drone_uuid='test-123')) self.assertEqual(response.remote_resource_uuid, "1351043") self.assertFalse(response.created - datetime.now() > timedelta(seconds=1)) self.assertFalse(response.updated - datetime.now() > timedelta(seconds=1)) self.mock_executor.return_value.run_command.assert_called_with( 'condor_submit -append "environment = TardisDroneUuid=test-123;TardisDroneCores=8;TardisDroneMemory=32768"' ' -a "request_cpus = 8" -a "request_memory = 32768" submit.jdl') self.mock_executor.reset()
def resource_attributes(self): return AttributeDict( machine_type='test2large', site_name='TestSite', remote_resource_uuid=4761849, resource_status=ResourceStatus.Booting, created=datetime.strptime("Wed Jan 23 2019 15:01:47", '%a %b %d %Y %H:%M:%S'), updated=datetime.strptime("Wed Jan 23 2019 15:02:17", '%a %b %d %Y %H:%M:%S'), drone_uuid='testsite-4761849')
def test_deploy_resource_w_submit_options(self): self.test_site_config.MachineTypeConfiguration.test2large.SubmitOptions = ( AttributeDict( short=AttributeDict(M="*****@*****.**"), long=AttributeDict(timeout=60), ) ) moab_adapter = MoabAdapter(machine_type="test2large", site_name="TestSite") run_async( moab_adapter.deploy_resource, resource_attributes=AttributeDict( machine_type="test2large", site_name="TestSite", ), ) self.mock_executor.return_value.run_command.assert_called_with( "msub -M [email protected] -j oe -m p -l walltime=02:00:00:00,mem=120gb,nodes=1:ppn=20 --timeout=60 startVM.py" # noqa: B950 )
def test_stop_resource(self): run_async( self.openstack_adapter.stop_resource, resource_attributes=AttributeDict( remote_resource_uuid="029312-1231-123123"), ) params = {"os-stop": None} self.mock_openstack_api.return_value.init_api.assert_called_with( timeout=60) self.mock_openstack_api.return_value.servers.run_action.assert_called_with( "029312-1231-123123", **params)
def test_terminate_resource(self): run_async( self.openstack_adapter.terminate_resource, resource_attributes=AttributeDict( remote_resource_uuid="029312-1231-123123"), ) self.mock_openstack_api.return_value.init_api.assert_called_with( timeout=60) self.mock_openstack_api.return_value.servers.force_delete.assert_called_with( "029312-1231-123123")
def test_drone_environment(self): self.site_adapter._configuration = AttributeDict( MachineMetaData=AttributeDict( test1xxl=AttributeDict(Cores=128, Memory=512, Disk=100))) self.site_adapter._machine_type = "test1xxl" self.assertEqual( AttributeDict(Cores=128, Memory=524288, Disk=104857600, Uuid="test-123"), self.site_adapter.drone_environment( drone_uuid="test-123", meta_data_translation_mapping=AttributeDict( Cores=1, Memory=1024, Disk=1024 * 1024, ), ), )
def test_handle_response_non_matching_wo_additional(self): test_response = {"other_test": 123} test_key_translator = {"new_test": "test"} test_translator_functions = {"test": str} self.assertEqual( self.site_adapter.handle_response(test_response, test_key_translator, test_translator_functions), AttributeDict(), )
def test_handle_response_matching(self): test_response = {"test": 123} test_key_translator = {"new_test": "test"} test_translator_functions = {"test": str} self.assertEqual( self.site_adapter.handle_response( test_response, test_key_translator, test_translator_functions ), AttributeDict(new_test="123"), ) self.assertEqual( self.site_adapter.handle_response( test_response, test_key_translator, test_translator_functions, additional="test123", ), AttributeDict(new_test="123", additional="test123"), )
def test_deploy_resource(self, mocked_aiohttp): self.assertEqual( run_async( self.kubernetes_adapter.deploy_resource, resource_attributes=AttributeDict( drone_uuid="testsite-089123", remote_resource_uuid="123456", obs_machine_meta_data_translation_mapping=AttributeDict( Cores=1, Memory=1024, Disk=1024 * 1024 ), ), ), AttributeDict( remote_resource_uuid="123456", drone_uuid="testsite-089123", resource_status=ResourceStatus.Booting, ), ) self.mock_kubernetes_api.return_value.create_namespaced_deployment.assert_called_with( # noqa: B950 namespace="default", body=self.body )
def test_resource_status_raise_past(self): # Update interval is 10 minutes, so set last update back by 11 minutes in order to execute sacct command and # creation date to 12 minutes ago past_timestamp = datetime.now() - timedelta(minutes=12) new_timestamp = datetime.now() - timedelta(minutes=11) self.moab_adapter._moab_status._last_update = new_timestamp response = run_async( self.moab_adapter.resource_status, AttributeDict(resource_id=1390065, remote_resource_uuid=1351043, created=past_timestamp)) self.assertEqual(response.resource_status, ResourceStatus.Stopped)
async def run(self, command, input=None, **kwargs): with self._multiplex_session(): if self.exception: raise self.exception if command.startswith("sleep"): _, duration = command.split() await asyncio.sleep(float(duration)) elif command != "Test": raise ValueError(f"Unsupported mock command: {command}") return AttributeDict( stdout=input and input.decode(), stderr="TestError", exit_status=0 )
def test_deploy_resource(self): response = run_async(self.adapter.deploy_resource, AttributeDict(drone_uuid="test-123")) self.assertEqual(response.remote_resource_uuid, "1351043") self.assertFalse( response.created - datetime.now() > timedelta(seconds=1)) self.assertFalse( response.updated - datetime.now() > timedelta(seconds=1)) self.mock_executor.return_value.run_command.assert_called_with( "condor_submit", stdin_input=CONDOR_SUBMIT_JDL) self.mock_executor.reset()
def test_resource_state_translation(self): for num, (_, state) in enumerate(STATE_TRANSLATIONS): job_id = f"76242{num:02}" return_resource_attributes = run_async( self.moab_adapter.resource_status, AttributeDict(remote_resource_uuid=job_id), ) self.assertEqual(return_resource_attributes.resource_status, state) self.mock_executor.return_value.run_command.assert_called_with( "showq --xml -w user=$(whoami) && showq -c --xml -w user=$(whoami)" )
def test_machine_meta_data(self): self.assertEqual( self.site_adapter.machine_meta_data, AttributeDict(Cores=128, Memory=512, Disk=100), ) # noinspection PyUnresolvedReferences del self.site_adapter._machine_type with self.assertRaises(AttributeError): # noinspection PyStatementEffect self.site_adapter.machine_meta_data
def setUp(self): self.test_path = os.path.dirname(os.path.realpath(__file__)) self.test_db = os.path.join(self.test_path, "test.db") try: os.remove(self.test_db) except FileNotFoundError: pass config = self.mock_config.return_value config.Plugins.SqliteRegistry.db_file = self.test_db config.Sites = [AttributeDict(name=self.test_site_name)] getattr(config, self.test_site_name).MachineTypes = [self.test_machine_type]
def test_machine_type_configuration(self): self.assertEqual( self.site_adapter.machine_type_configuration, AttributeDict(test_id="abc123"), ) # noinspection PyUnresolvedReferences del self.site_adapter._machine_type with self.assertRaises(AttributeError): # noinspection PyStatementEffect self.site_adapter.machine_type_configuration