def setUp(self): """ It creates the memory db """ db.create_all() # We store some Applications in the db for the tests application_1 = Application() application_1.name = 'AppName_1' application_2 = Application() application_2.name = 'AppName_2' # Adding executing scripts execution_script_1 = ExecutionConfiguration() execution_script_1.execution_type = "slurm:sbatch" execution_script_2 = ExecutionConfiguration() execution_script_2.execution_type = "slurm:sbatch2" application_2.execution_configurations = [ execution_script_1, execution_script_2 ] db.session.add(application_1) db.session.add(application_2) # We store some testbeds in the db for the tests testbed_1 = Testbed("name_1", True, "slurm", "ssh", "user@server", ['slurm']) testbed_2 = Testbed("name_2", False, "slurm", "ssh", "user@server", ['slurm']) testbed_3 = Testbed("name_3", True, "slurm", "ssh", "user@server", ['slurm', 'slurm:singularity']) db.session.add(testbed_1) db.session.add(testbed_2) db.session.add(testbed_3) db.session.commit() deployment = Deployment() deployment.executable_id = execution_script_1.id deployment.testbed_id = testbed_1.id db.session.add(deployment) # We store some nodes in the db for the tests node_1 = Node() node_1.name = "node_1" node_1.information_retrieved = True node_2 = Node() node_2.name = "node_2" node_2.information_retrieved = False db.session.add(node_1) db.session.add(node_2) execution = Execution() execution.execution_type = "execution_type" execution.status = "status" db.session.add(execution) db.session.commit()
def test_patch_execution_script_preprocessor(self, mock_execute_application): """ Verifies the correct work of the function. """ # First we verify that nothing happens if launch_execution = False data = {'launch_execution': False} response = self.client.patch("/api/v1/execution_configurations/1", data=json.dumps(data), content_type='application/json') self.assertEquals(200, response.status_code) execution_script = response.json self.assertEquals("slurm:sbatch", execution_script['execution_type']) """ If the execution_script has not assigned a testbed we give an error returning a 409 Conflict in the resource """ data = {'launch_execution': True} response = self.client.patch("/api/v1/execution_configurations/1", data=json.dumps(data), content_type='application/json') self.assertEquals(409, response.status_code) execution_script = response.json self.assertEquals( 'No deployment configured to execute the application', response.json['message']) """ Now we have an off-line testbed testbed to submit the execution """ testbed = Testbed("name", False, "slurm", "ssh", "user@server", ['slurm']) db.session.add(testbed) executable = Executable() executable.source_code_file = 'source_code_file' executable.compilation_script = 'compilation_script' executable.compilation_type = 'compilation_type' db.session.add(executable) db.session.commit() execution_script = db.session.query(ExecutionConfiguration).filter_by(id=1).first() execution_script.testbed = testbed execution_script.executable = executable db.session.commit() deployment = Deployment() deployment.executable_id = executable.id deployment.testbed_id = testbed.id db.session.add(deployment) db.session.commit() data = {'launch_execution': True} response = self.client.patch("/api/v1/execution_configurations/1", data=json.dumps(data), content_type='application/json') self.assertEquals(403, response.status_code) self.assertEquals( 'Testbed does not allow on-line connection', response.json['message']) """ Now we are able to launch the execution """ testbed.on_line = True db.session.commit() data = {'launch_execution': True} response = self.client.patch("/api/v1/execution_configurations/1", data=json.dumps(data), content_type='application/json') self.assertEquals(200, response.status_code) """ Now we are able to lauch the execution with create_profile= True """ data = { 'launch_execution': True, 'create_profile': True } response = self.client.patch("/api/v1/execution_configurations/1", data=json.dumps(data), content_type='application/json') """ Now we are able to lauch the execution with use_storaged_profile=true and create_profile=True """ data = { 'launch_execution': True, 'create_profile': True, 'use_storaged_profile': True } response = self.client.patch("/api/v1/execution_configurations/1", data=json.dumps(data), content_type='application/json') """ Now we are able to lauch the execution with use_storaged_profile=true and Execution_Configuration without any profile storaged on it. """ data = { 'launch_execution': True, 'use_storaged_profile': True } response = self.client.patch("/api/v1/execution_configurations/1", data=json.dumps(data), content_type='application/json') """ Now we are able to lauch the execution with use_storaged_profile=true and Execution_Configuration with a profile storaged on it. """ data = { 'launch_execution': True, 'use_storaged_profile': True } execution_script = db.session.query(ExecutionConfiguration).filter_by(id=1).first() execution_script.profile_file = 'pepito.profile' db.session.commit() response = self.client.patch("/api/v1/execution_configurations/1", data=json.dumps(data), content_type='application/json') call_1 = call(execution_script, False, False) call_2 = call(execution_script, True, False) call_3 = call(execution_script, True, False) call_4 = call(execution_script, False, False) call_5 = call(execution_script, False, True) calls = [ call_1, call_2, call_3, call_4, call_5 ] mock_execute_application.assert_has_calls(calls)
def test_execute_application_type_torque_qsub(self, mock_shell, mock_add_nodes): """ It verifies that the application type slurm sbatch is executed """ # First we verify that the testbed is of type TORQUE to be able # to execute it, in this case it should give an error since it is # not of type torque # We define the different entities necessary for the test. testbed = Testbed( name="nova2", on_line=True, category="xxxx", protocol="SSH", endpoint="*****@*****.**", package_formats=['sbatch', 'SINGULARITY'], extra_config={ "enqueue_compss_sc_cfg": "nova.cfg", "enqueue_env_file": "/home_nfs/home_ejarquej/installations/rc1707/COMPSs/compssenv" }) db.session.add(testbed) application = Application(name="super_app") db.session.add(application) db.session.commit() # So application and testbed get an id executable = Executable() executable.compilation_type = Executable.__type_torque_qsub__ executable.executable_file = "pepito.sh" db.session.add(executable) db.session.commit() # We do this so executable gets and id deployment = Deployment() deployment.testbed_id = testbed.id deployment.executable_id = executable.id db.session.add( deployment) # We add the executable to the db so it has an id execution_config = ExecutionConfiguration() execution_config.execution_type = Executable.__type_torque_qsub__ execution_config.application = application execution_config.testbed = testbed execution_config.executable = executable db.session.add(execution_config) db.session.commit() execution = Execution() execution.execution_type = Executable.__type_torque_qsub__ execution.status = Execution.__status_submitted__ torque.execute_batch(execution, execution_config.id) self.assertEquals(Execution.__status_failed__, execution.status) self.assertEquals("Testbed does not support TORQUE:QSUB applications", execution.output) # If the testbed is off-line, execution isn't allowed also testbed.category = Testbed.torque_category testbed.on_line = False db.session.commit() execution = Execution() execution.execution_type = Executable.__type_torque_qsub__ execution.status = Execution.__status_submitted__ torque.execute_batch(execution, execution_config.id) self.assertEquals(Executable.__type_torque_qsub__, execution.execution_type) self.assertEquals(Execution.__status_failed__, execution.status) self.assertEquals("Testbed is off-line", execution.output) ## Test executing output = b'1208.cloudserver' mock_shell.return_value = output testbed.category = Testbed.torque_category testbed.on_line = True db.session.commit() execution = Execution() execution.execution_type = Executable.__type_torque_qsub__ execution.status = Execution.__status_submitted__ torque.execute_batch(execution, execution_config.id) mock_shell.assert_called_with("qsub", "*****@*****.**", ["pepito.sh"]) execution = db.session.query(Execution).filter_by( execution_configuration_id=execution_config.id).first() self.assertEqual(execution.execution_type, execution_config.execution_type) self.assertEqual(execution.status, Execution.__status_running__) self.assertEqual("1208.cloudserver", execution.batch_id)
def test_execute_srun(self, mock_shell): """ Verifyies the correct work of the function "execute_srun" """ # We define the different entities necessaryb for the test. testbed = Testbed( name="nova2", on_line=True, category="SLURM", protocol="SSH", endpoint="*****@*****.**", package_formats=['sbatch', 'SINGULARITY'], extra_config={ "enqueue_compss_sc_cfg": "nova.cfg", "enqueue_env_file": "/home_nfs/home_ejarquej/installations/rc1707/COMPSs/compssenv" }) application = Application(name="super_app") executable = Executable() executable.source_code_file = 'test.zip' executable.compilation_script = 'gcc -X' executable.compilation_type = "SINGULARITY:SRUN" executable.singularity_app_folder = "/singularity/app/folder" executable.singularity_image_file = "pepito.img" executable.status = "COMPILED" executable.application = application deployment = Deployment() deployment.testbed_id = testbed.id deployment.executable_id = executable.id deployment.path = "/pepito/pepito.img" execution_config = ExecutionConfiguration() execution_config.execution_type = "SINGULARITY:SRUN" execution_config.application = application execution_config.testbed = testbed execution_config.executable = executable execution_config.num_nodes = 2 #execution_config.num_gpus_per_node = 2 execution_config.num_cpus_per_node = 16 execution_config.exec_time = 10 execution_config.command = "/apps/application/master/Matmul 2 1024 12.34 /home_nfs/home_ejarquej/demo_test/cpu_gpu_run_data" execution_config.compss_config = "--worker_in_master_cpus=12 --worker_in_master_memory=24000 --worker_working_dir=/home_nfs/home_ejarquej --lang=c --monitoring=1000 -d" output = b' JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON)\n 4610 all singular garciad R 0:01 2 ns[55-56]\n' mock_shell.return_value = output # TEST starts here: output_srun = slurm.execute_srun(testbed, execution_config, executable, deployment, True) call_1 = call('(', "*****@*****.**", [ "srun", "-N", "2", "-n", "16", "singularity", "run", "/pepito/pepito.img", ">", "allout.txt", "2>&1", "&", ")", ";", "sleep", "1;", "squeue" ]) # adding a new type of execution execution_config = ExecutionConfiguration() execution_config.execution_type = "SINGULARITY:SRUN" execution_config.application = application execution_config.testbed = testbed execution_config.executable = executable execution_config.num_gpus_per_node = 2 execution_config.num_cpus_per_node = 16 execution_config.exec_time = 10 execution_config.command = "/apps/application/master/Matmul 2 1024 12.34 /home_nfs/home_ejarquej/demo_test/cpu_gpu_run_data" execution_config.compss_config = "--worker_in_master_cpus=12 --worker_in_master_memory=24000 --worker_working_dir=/home_nfs/home_ejarquej --lang=c --monitoring=1000 -d" self.assertEquals(output, output_srun) output_srun = slurm.execute_srun(testbed, execution_config, executable, deployment, True) call_2 = call('(', "*****@*****.**", [ "srun", "--gres=gpu:2", "-n", "16", "singularity", "run", "/pepito/pepito.img", ">", "allout.txt", "2>&1", "&", ")", ";", "sleep", "1;", "squeue" ]) calls = [call_1, call_2] mock_shell.assert_has_calls(calls) self.assertEquals(output, output_srun)