def setUp(self):
async def mock_set_state(drone, state):
drone.state = state
f = asyncio.Future()
f.set_result(None)
return f
self.drone = self.mock_drone.return_value
self.drone.resource_attributes = AttributeDict(
drone_uuid="test-923ABF",
remote_resource_uuid="0815",
updated=datetime.now() - timedelta(minutes=10),
)
self.drone.demand = 8.0
self.drone._supply = 8.0
self.drone.minimum_lifetime = 3600
self.drone.set_state.side_effect = partial(mock_set_state, self.drone)
self.drone.site_agent.deploy_resource.return_value = async_return(
return_value=AttributeDict()
)
self.drone.site_agent.resource_status.return_value = async_return(
return_value=AttributeDict()
)
self.drone.site_agent.stop_resource.return_value = async_return(
return_value=AttributeDict()
)
self.drone.site_agent.terminate_resource.return_value = async_return(
return_value=AttributeDict()
)
self.drone.batch_system_agent.integrate_machine.return_value = async_return(
return_value=None
)
self.drone.batch_system_agent.disintegrate_machine.return_value = async_return(
return_value=None
)
self.drone.batch_system_agent.drain_machine.return_value = async_return(
return_value=None
)
self.drone.batch_system_agent.get_machine_status.return_value = async_return(
return_value=None
)
self.drone.batch_system_agent.get_allocation.return_value = async_return(
return_value=None
)
self.drone.batch_system_agent.get_utilisation.return_value = async_return(
return_value=None
)
def test_resource_status(self):
self.assertEqual(
run_async(
self.openstack_adapter.resource_status,
resource_attributes=AttributeDict(
remote_resource_uuid="029312-1231-123123"),
),
AttributeDict(
drone_uuid="testsite-089123",
remote_resource_uuid="029312-1231-123123",
resource_status=ResourceStatus.Running,
),
)
self.mock_openstack_api.return_value.init_api.assert_called_with(
timeout=60)
self.mock_openstack_api.return_value.servers.get.assert_called_with(
"029312-1231-123123")
def resource_attributes(self):
return AttributeDict(
machine_type="test2large",
site_name="TestSite",
remote_resource_uuid=1390065,
resource_status=ResourceStatus.Booting,
drone_uuid="testsite-1390065",
)
def test_resource_status_raise_future(self):
future_timestamp = datetime.now() + timedelta(minutes=1)
with self.assertRaises(TardisResourceStatusUpdateFailed):
with self.assertLogs(logging.getLogger(), logging.ERROR):
run_async(
self.adapter.resource_status,
AttributeDict(remote_resource_uuid="1351043",
created=future_timestamp))
def wrapper(self):
executor = self.mock_executor.return_value
executor.run_command.return_value = async_return(
return_value=AttributeDict(
stdout=stdout, stderr=stderr, exit_code=exit_code))
executor.run_command.side_effect = raise_exception
func(self)
executor.run_command.side_effect = None
def test_terminate_resource(self):
run_async(
self.cloudstack_adapter.terminate_resource,
resource_attributes=AttributeDict(remote_resource_uuid="123456"),
)
self.mock_cloudstack_api.return_value.destroyVirtualMachine.assert_called_with(
id="123456")
def test_deploy_resource(self):
self.assertEqual(
run_async(
self.openstack_adapter.deploy_resource,
resource_attributes=AttributeDict(drone_uuid="testsite-089123"),
),
AttributeDict(drone_uuid="testsite-089123"),
)
self.mock_openstack_api.return_value.init_api.assert_called_with(timeout=60)
self.mock_openstack_api.return_value.servers.create.assert_called_with(
server={
"imageRef": "bc613271-6a54-48ca-9222-47e009dc0c29",
"name": "testsite-089123",
}
)
def run_the_matrix(self, matrix, initial_state):
for resource_status, machine_status, new_state in matrix:
self.drone.site_agent.resource_status.return_value = async_return(
return_value=AttributeDict(resource_status=resource_status))
self.drone.batch_system_agent.get_machine_status.return_value = async_return(
return_value=machine_status)
self.drone.state.return_value = initial_state
run_async(self.drone.state.return_value.run, self.drone)
self.assertIsInstance(self.drone.state, new_state)
def test_resource_status_raise_past(self):
# Update interval is 10 minutes, so set last update back by 11 minutes in order to execute condor_q command and
# creation date to 12 minutes ago
past_timestamp = datetime.now() - timedelta(minutes=12)
self.adapter._htcondor_queue._last_update = datetime.now() - timedelta(minutes=11)
with self.assertLogs(logging.getLogger(), logging.ERROR):
response = run_async(self.adapter.resource_status, AttributeDict(remote_resource_uuid="1351043",
created=past_timestamp))
self.assertEqual(response.resource_status, ResourceStatus.Deleted)
def get_test_param(drone_uuid, resource_status):
return AttributeDict(
site_name="test-site",
machine_type="test_machine_type",
created=datetime.now(),
updated=datetime.now(),
drone_uuid=drone_uuid,
resource_status=resource_status,
)
def test_update_configuration(self):
with open(os.path.join(self.test_path, "OpenStack.yml"),
"r") as config_file:
config_file_content = yaml.safe_load(config_file)
self.configuration1 = Configuration(config_file_content)
self.assertEqual(
self.configuration1.OpenStack,
AttributeDict(api_key="qwertzuiop", api_secret="katze123"),
)
def mocked_get_user(user_name):
if user_name == "test":
return AttributeDict(
user_name="test",
hashed_password=
"******", # noqa B509
scopes=["resources:get"],
)
return None
def setUp(self) -> None:
self.mock_site_agent.machine_meta_data = AttributeDict(Cores=8)
self.mock_site_agent.drone_minimum_lifetime = None
self.mock_plugin = MagicMock(spec=Plugin)()
self.mock_plugin.notify.return_value = async_return()
self.drone = Drone(
site_agent=self.mock_site_agent,
batch_system_agent=self.mock_batch_system_agent,
)
def test_get_utilisation(self):
self.assertEqual(
run_async(self.fake_adapter.get_utilisation, "test-123"), 1.0)
self.config.BatchSystem.utilisation = AttributeDict(
get_value=lambda: 0.9)
self.fake_adapter = FakeBatchSystemAdapter()
self.assertEqual(
run_async(self.fake_adapter.get_utilisation, "test-123"), 0.9)
def test_deploy_resource(self):
response = run_async(self.adapter.deploy_resource, AttributeDict(drone_uuid='test-123'))
self.assertEqual(response.remote_resource_uuid, "1351043")
self.assertFalse(response.created - datetime.now() > timedelta(seconds=1))
self.assertFalse(response.updated - datetime.now() > timedelta(seconds=1))
self.mock_executor.return_value.run_command.assert_called_with(
'condor_submit -append "environment = TardisDroneUuid=test-123;TardisDroneCores=8;TardisDroneMemory=32768"'
' -a "request_cpus = 8" -a "request_memory = 32768" submit.jdl')
self.mock_executor.reset()
def resource_attributes(self):
return AttributeDict(
machine_type='test2large',
site_name='TestSite',
remote_resource_uuid=4761849,
resource_status=ResourceStatus.Booting,
created=datetime.strptime("Wed Jan 23 2019 15:01:47",
'%a %b %d %Y %H:%M:%S'),
updated=datetime.strptime("Wed Jan 23 2019 15:02:17",
'%a %b %d %Y %H:%M:%S'),
drone_uuid='testsite-4761849')
def test_deploy_resource_w_submit_options(self):
self.test_site_config.MachineTypeConfiguration.test2large.SubmitOptions = (
AttributeDict(
short=AttributeDict(M="*****@*****.**"),
long=AttributeDict(timeout=60),
)
)
moab_adapter = MoabAdapter(machine_type="test2large", site_name="TestSite")
run_async(
moab_adapter.deploy_resource,
resource_attributes=AttributeDict(
machine_type="test2large",
site_name="TestSite",
),
)
self.mock_executor.return_value.run_command.assert_called_with(
"msub -M [email protected] -j oe -m p -l walltime=02:00:00:00,mem=120gb,nodes=1:ppn=20 --timeout=60 startVM.py" # noqa: B950
)
def test_stop_resource(self):
run_async(
self.openstack_adapter.stop_resource,
resource_attributes=AttributeDict(
remote_resource_uuid="029312-1231-123123"),
)
params = {"os-stop": None}
self.mock_openstack_api.return_value.init_api.assert_called_with(
timeout=60)
self.mock_openstack_api.return_value.servers.run_action.assert_called_with(
"029312-1231-123123", **params)
def test_terminate_resource(self):
run_async(
self.openstack_adapter.terminate_resource,
resource_attributes=AttributeDict(
remote_resource_uuid="029312-1231-123123"),
)
self.mock_openstack_api.return_value.init_api.assert_called_with(
timeout=60)
self.mock_openstack_api.return_value.servers.force_delete.assert_called_with(
"029312-1231-123123")
def test_drone_environment(self):
self.site_adapter._configuration = AttributeDict(
MachineMetaData=AttributeDict(
test1xxl=AttributeDict(Cores=128, Memory=512, Disk=100)))
self.site_adapter._machine_type = "test1xxl"
self.assertEqual(
AttributeDict(Cores=128,
Memory=524288,
Disk=104857600,
Uuid="test-123"),
self.site_adapter.drone_environment(
drone_uuid="test-123",
meta_data_translation_mapping=AttributeDict(
Cores=1,
Memory=1024,
Disk=1024 * 1024,
),
),
)
def test_handle_response_non_matching_wo_additional(self):
test_response = {"other_test": 123}
test_key_translator = {"new_test": "test"}
test_translator_functions = {"test": str}
self.assertEqual(
self.site_adapter.handle_response(test_response,
test_key_translator,
test_translator_functions),
AttributeDict(),
)
def test_handle_response_matching(self):
test_response = {"test": 123}
test_key_translator = {"new_test": "test"}
test_translator_functions = {"test": str}
self.assertEqual(
self.site_adapter.handle_response(
test_response, test_key_translator, test_translator_functions
),
AttributeDict(new_test="123"),
)
self.assertEqual(
self.site_adapter.handle_response(
test_response,
test_key_translator,
test_translator_functions,
additional="test123",
),
AttributeDict(new_test="123", additional="test123"),
)
def test_deploy_resource(self, mocked_aiohttp):
self.assertEqual(
run_async(
self.kubernetes_adapter.deploy_resource,
resource_attributes=AttributeDict(
drone_uuid="testsite-089123",
remote_resource_uuid="123456",
obs_machine_meta_data_translation_mapping=AttributeDict(
Cores=1, Memory=1024, Disk=1024 * 1024
),
),
),
AttributeDict(
remote_resource_uuid="123456",
drone_uuid="testsite-089123",
resource_status=ResourceStatus.Booting,
),
)
self.mock_kubernetes_api.return_value.create_namespaced_deployment.assert_called_with( # noqa: B950
namespace="default", body=self.body
)
def test_resource_status_raise_past(self):
# Update interval is 10 minutes, so set last update back by 11 minutes in order to execute sacct command and
# creation date to 12 minutes ago
past_timestamp = datetime.now() - timedelta(minutes=12)
new_timestamp = datetime.now() - timedelta(minutes=11)
self.moab_adapter._moab_status._last_update = new_timestamp
response = run_async(
self.moab_adapter.resource_status,
AttributeDict(resource_id=1390065,
remote_resource_uuid=1351043,
created=past_timestamp))
self.assertEqual(response.resource_status, ResourceStatus.Stopped)
async def run(self, command, input=None, **kwargs):
with self._multiplex_session():
if self.exception:
raise self.exception
if command.startswith("sleep"):
_, duration = command.split()
await asyncio.sleep(float(duration))
elif command != "Test":
raise ValueError(f"Unsupported mock command: {command}")
return AttributeDict(
stdout=input and input.decode(), stderr="TestError", exit_status=0
)
def test_deploy_resource(self):
response = run_async(self.adapter.deploy_resource,
AttributeDict(drone_uuid="test-123"))
self.assertEqual(response.remote_resource_uuid, "1351043")
self.assertFalse(
response.created - datetime.now() > timedelta(seconds=1))
self.assertFalse(
response.updated - datetime.now() > timedelta(seconds=1))
self.mock_executor.return_value.run_command.assert_called_with(
"condor_submit", stdin_input=CONDOR_SUBMIT_JDL)
self.mock_executor.reset()
def test_resource_state_translation(self):
for num, (_, state) in enumerate(STATE_TRANSLATIONS):
job_id = f"76242{num:02}"
return_resource_attributes = run_async(
self.moab_adapter.resource_status,
AttributeDict(remote_resource_uuid=job_id),
)
self.assertEqual(return_resource_attributes.resource_status, state)
self.mock_executor.return_value.run_command.assert_called_with(
"showq --xml -w user=$(whoami) && showq -c --xml -w user=$(whoami)"
)
def test_machine_meta_data(self):
self.assertEqual(
self.site_adapter.machine_meta_data,
AttributeDict(Cores=128, Memory=512, Disk=100),
)
# noinspection PyUnresolvedReferences
del self.site_adapter._machine_type
with self.assertRaises(AttributeError):
# noinspection PyStatementEffect
self.site_adapter.machine_meta_data
def setUp(self):
self.test_path = os.path.dirname(os.path.realpath(__file__))
self.test_db = os.path.join(self.test_path, "test.db")
try:
os.remove(self.test_db)
except FileNotFoundError:
pass
config = self.mock_config.return_value
config.Plugins.SqliteRegistry.db_file = self.test_db
config.Sites = [AttributeDict(name=self.test_site_name)]
getattr(config, self.test_site_name).MachineTypes = [self.test_machine_type]
def test_machine_type_configuration(self):
self.assertEqual(
self.site_adapter.machine_type_configuration,
AttributeDict(test_id="abc123"),
)
# noinspection PyUnresolvedReferences
del self.site_adapter._machine_type
with self.assertRaises(AttributeError):
# noinspection PyStatementEffect
self.site_adapter.machine_type_configuration
