def test_compile_executables(self, mock_compiler):
""" Test the compilation procedures """
# Changing the config file path for the running of the test.
config.COMPILATION_CONFIG_FILE = "compilation_config.json"
executable_1 = Executable()
executable_1.source_code_file = 'source1'
executable_1.compilation_script = 'script1'
executable_1.compilation_type = "type1"
executable_2 = Executable()
executable_2.source_code_file = 'source2'
executable_2.compilation_script = 'script2'
executable_2.compilation_type = "SINGULARITY:PM"
db.session.add(executable_1)
db.session.add(executable_2)
db.session.commit()
# We perform the action
compiler.compile_executables('asdf')
# At the end fo the test we should have those status:
self.assertEquals(executable_1.status, Executable.__status_error_type__)
self.assertEquals(executable_2.status, Executable.__status_compiling__)
# We verify the mock was called:
mock_compiler.assert_called_with(executable_2, 'asdf')
def test_crud_executable(self):
"""It tests basis CRUD operations of an Executable Class"""
# We verify that the object is not in the db after creating it
executable = Executable()
executable.source_code_file = "source"
executable.compilation_script = "script"
executable.compilation_type = "type"
self.assertIsNone(executable.id)
# We store the object in the db
db.session.add(executable)
db.session.commit()
# We recover the executable form the db
executable = db.session.query(Executable).filter_by(
id=executable.id).first()
self.assertEquals("source", executable.source_code_file)
# We check that we can update the Executable
executable.executable_file = 'pepito'
executable.singularity_app_folder = "app_folder"
db.session.commit()
executable = db.session.query(Executable).filter_by(
id=executable.id).first()
self.assertEquals('pepito', executable.executable_file)
self.assertEquals('app_folder', executable.singularity_app_folder)
# We check that we can delete the Executable
db.session.delete(executable)
db.session.commit()
count = db.session.query(Executable).count()
self.assertEquals(0, count)
def test_return_not_compiled_executable(self):
"""It checks the method that returns all the no compiled executables"""
executable_1 = Executable()
executable_1.source_code_file = 'source1'
executable_1.compilation_script = 'script1'
executable_1.compilation_type = "type1"
executable_2 = Executable()
executable_2.source_code_file = 'source2'
executable_2.compilation_script = 'script2'
executable_2.compilation_type = "type2"
executable_2.status = 'pepito'
db.session.add(executable_1)
db.session.add(executable_2)
db.session.commit()
executables = compiler.return_not_compiled_executables()
self.assertEquals(1, len(executables))
self.assertEquals('NOT_COMPILED', executables[0].status)
self.assertEquals("source1", executables[0].source_code_file)
def test_compile_singularity_pm(self, mock_random_folder, mock_upload_zip,
mock_unzip_src, mock_create_sing_template,
mock_image, mock_build):
""" Test that the right workflow is managed to create a container """
config.COMPILATION_CONFIG_FILE = "compilation_config.json"
configuration = config.find_compilation_config('SINGULARITY:PM')
mock_random_folder.return_value = 'dest_folder'
mock_create_sing_template.return_value = 'template.def'
mock_build.return_value = '/tmp/image.img'
executable = Executable()
executable.source_code_file = "test.zip"
executable.compilation_script = "xxxx"
executable.compilation_type = "xxxx"
db.session.add(executable)
db.session.commit()
compiler.compile_singularity_pm(
executable,
'asdf') # TODO Create a executable when the method evolvers.
# We verify that a random folder was created
mock_random_folder.assert_called_with('ubuntu@localhost:2222')
# We verfiy that the src file is uploaded to the random folder
mock_upload_zip.assert_called_with(executable, 'ubuntu@localhost:2222',
'dest_folder', 'asdf')
# We verify that the src file is uncrompress
mock_unzip_src.assert_called_with(executable, 'ubuntu@localhost:2222',
'dest_folder')
# We verify that the creation of the template is done
mock_create_sing_template.assert_called_with(configuration, executable,
'ubuntu@localhost:2222',
'dest_folder')
# We verify that the image was created
mock_image.assert_called_with(configuration, 'ubuntu@localhost:2222',
'singularity_pm.img')
# We verify that the container was build
mock_build.assert_called_with('ubuntu@localhost:2222',
'template.def',
'singularity_pm.img',
'asdf',
become=True)
executable = db.session.query(Executable).filter_by(
status=Executable.__status_compiled__).first()
self.assertEquals('test.zip', executable.source_code_file)
self.assertEquals('COMPILED', executable.status)
self.assertEquals('/tmp/image.img', executable.singularity_image_file)
def test_unzip_src(self, mock_shell_execute):
"""
It verifies the workflow of unziping the zip file
"""
executable = Executable()
executable.source_code_file = "test.zip"
executable.compilation_script = "xxxx"
executable.compilation_type = "xxxx"
compiler.unzip_src(executable, '*****@*****.**', '/home/pepito')
zip_file = os.path.join('/home/pepito', 'test.zip')
mock_shell_execute.assert_called_with('unzip', '*****@*****.**', [ zip_file, '-d', '/home/pepito'])
def test_initialization_executable(self):
"""Test the initialization of the object"""
executable = Executable()
executable.source_code_file = "source"
executable.compilation_script = "script"
executable.compilation_type = "type"
executable.singularity_app_folder = "app_folder"
executable.singularity_image_file = "image.img"
executable.status = "NOT_COMPILED"
self.assertEquals("source", executable.source_code_file)
self.assertEquals("script", executable.compilation_script)
self.assertEquals("type", executable.compilation_type)
self.assertEquals("NOT_COMPILED", executable.status)
self.assertEquals("app_folder", executable.singularity_app_folder)
self.assertEquals("image.img", executable.singularity_image_file)
def test_upload_zip_file_application(self, mock_scp, mock_exec):
"""
Test the function of uploading a zip file of the application
to the selected testbed in an specific folder
"""
executable = Executable()
executable.source_code_file = "test.zip"
executable.compilation_script = "xxxx"
executable.compilation_type = "xxxx"
compiler.upload_zip_file_application(executable, '*****@*****.**', 'dest_folder', '/tmp')
mock_scp.assert_called_with(os.path.join('/tmp', 'test.zip'), '*****@*****.**', './dest_folder')
compiler.upload_zip_file_application(executable, '', 'dest_folder', '/tmp')
mock_exec.assert_called_with('cp', params=[ os.path.join('/tmp', 'test.zip'), './dest_folder'])
def test_create_singularity_template(self, mock_template, mock_scp):
"""
It test the correct work of the function craete singularity template
"""
configuration = { 'singularity_template': 'sing_template'}
executable = Executable()
executable.source_code_file = "test.zip"
executable.compilation_script = 'comp_script'
executable.compilation_type = "xxxx"
mock_template.return_value = 'sing_template'
template = compiler.create_singularity_template(configuration, executable, '*****@*****.**', 'comp_folder')
self.assertEquals('sing_template', template)
mock_template.assert_called_with('sing_template', 'comp_script', 'comp_folder')
mock_scp.assert_called_with('sing_template', '*****@*****.**', '.')
def upload_application(app_id):
application = db.session.query(Application).filter_by(id=app_id).first()
upload_folder = app.config['APP_FOLDER']
compilation_type = request.args.get('compilation_type')
compilation_script = request.args.get('compilation_script')
if not application:
return "Application with id: " + str(
app_id) + " does not exists in the database"
elif not compilation_type:
return "It is necessary to specify a compilation_type query param"
elif not compilation_script:
return "It is necessary to specify a compilation_script query param"
else:
# We define a filename
filename_uuid = uuid.uuid4()
if 'file' not in request.files:
return "No file specified"
print(request.files)
file = request.files['file']
if file.filename == '':
return "No file specified"
if file and allowed_file(file.filename):
filename = str(uuid.uuid4()) + ".zip"
file.save(os.path.join(upload_folder, filename))
# We store the compilation/executable information in the db
executable = Executable()
executable.source_code_file = filename
executable.compilation_script = compilation_script
executable.compilation_type = compilation_type
application.executables.append(executable)
db.session.commit()
return "file upload for app with id: " + str(app_id)
else:
return "file type not supported"
def test_patch_execution_script_preprocessor(self, mock_execute_application):
"""
Verifies the correct work of the function.
"""
# First we verify that nothing happens if launch_execution = False
data = {'launch_execution': False}
response = self.client.patch("/api/v1/execution_configurations/1",
data=json.dumps(data),
content_type='application/json')
self.assertEquals(200, response.status_code)
execution_script = response.json
self.assertEquals("slurm:sbatch", execution_script['execution_type'])
"""
If the execution_script has not assigned a testbed we give an error
returning a 409 Conflict in the resource
"""
data = {'launch_execution': True}
response = self.client.patch("/api/v1/execution_configurations/1",
data=json.dumps(data),
content_type='application/json')
self.assertEquals(409, response.status_code)
execution_script = response.json
self.assertEquals(
'No deployment configured to execute the application',
response.json['message'])
"""
Now we have an off-line testbed testbed to submit the execution
"""
testbed = Testbed("name", False, "slurm", "ssh", "user@server", ['slurm'])
db.session.add(testbed)
executable = Executable()
executable.source_code_file = 'source_code_file'
executable.compilation_script = 'compilation_script'
executable.compilation_type = 'compilation_type'
db.session.add(executable)
db.session.commit()
execution_script = db.session.query(ExecutionConfiguration).filter_by(id=1).first()
execution_script.testbed = testbed
execution_script.executable = executable
db.session.commit()
deployment = Deployment()
deployment.executable_id = executable.id
deployment.testbed_id = testbed.id
db.session.add(deployment)
db.session.commit()
data = {'launch_execution': True}
response = self.client.patch("/api/v1/execution_configurations/1",
data=json.dumps(data),
content_type='application/json')
self.assertEquals(403, response.status_code)
self.assertEquals(
'Testbed does not allow on-line connection',
response.json['message'])
"""
Now we are able to launch the execution
"""
testbed.on_line = True
db.session.commit()
data = {'launch_execution': True}
response = self.client.patch("/api/v1/execution_configurations/1",
data=json.dumps(data),
content_type='application/json')
self.assertEquals(200, response.status_code)
"""
Now we are able to lauch the execution with create_profile= True
"""
data = { 'launch_execution': True, 'create_profile': True }
response = self.client.patch("/api/v1/execution_configurations/1",
data=json.dumps(data),
content_type='application/json')
"""
Now we are able to lauch the execution with use_storaged_profile=true and create_profile=True
"""
data = { 'launch_execution': True,
'create_profile': True,
'use_storaged_profile': True }
response = self.client.patch("/api/v1/execution_configurations/1",
data=json.dumps(data),
content_type='application/json')
"""
Now we are able to lauch the execution with use_storaged_profile=true
and Execution_Configuration without any profile storaged on it.
"""
data = { 'launch_execution': True,
'use_storaged_profile': True }
response = self.client.patch("/api/v1/execution_configurations/1",
data=json.dumps(data),
content_type='application/json')
"""
Now we are able to lauch the execution with use_storaged_profile=true
and Execution_Configuration with a profile storaged on it.
"""
data = { 'launch_execution': True,
'use_storaged_profile': True }
execution_script = db.session.query(ExecutionConfiguration).filter_by(id=1).first()
execution_script.profile_file = 'pepito.profile'
db.session.commit()
response = self.client.patch("/api/v1/execution_configurations/1",
data=json.dumps(data),
content_type='application/json')
call_1 = call(execution_script, False, False)
call_2 = call(execution_script, True, False)
call_3 = call(execution_script, True, False)
call_4 = call(execution_script, False, False)
call_5 = call(execution_script, False, True)
calls = [ call_1, call_2, call_3, call_4, call_5 ]
mock_execute_application.assert_has_calls(calls)
def test_post_deployment_preprocessor(self, mock_upload_deployment):
"""
It checks the correct work of the post_deployment_preprocessor function
"""
# Error raised becasue missing testbed_id
data = { 'asdf' : 'asdf'}
response = self.client.post("/api/v1/deployments",
data=json.dumps(data),
content_type='application/json')
self.assertEquals(400, response.status_code)
self.assertEquals('Missing testbed_id field in the inputed JSON', response.json['message'])
# Error raised because missing executable_di
data = { 'asdf' : 'asdf',
'testbed_id' : 1 }
response = self.client.post("/api/v1/deployments",
data=json.dumps(data),
content_type='application/json')
self.assertEquals(400, response.status_code)
self.assertEquals('Missing executable_id field in the inputed JSON', response.json['message'])
# Error raised because the testbed_id is wrong
data = { 'executable_id' : 1,
'testbed_id' : 5 }
response = self.client.post("/api/v1/deployments",
data=json.dumps(data),
content_type='application/json')
self.assertEquals(400, response.status_code)
self.assertEquals('No testbed found with that id in the database', response.json['message'])
# Error raised because the executable_id is wrong
data = { 'executable_id' : 1,
'testbed_id' : 1 }
response = self.client.post("/api/v1/deployments",
data=json.dumps(data),
content_type='application/json')
self.assertEquals(400, response.status_code)
self.assertEquals('No executable found with that id in the database', response.json['message'])
# Error raised because the testbed is off-line
# We verify that the object is not in the db after creating it
executable = Executable()
executable.source_code_file = 'source'
executable.compilation_script = 'script'
executable.compilation_type = 'type'
db.session.add(executable)
db.session.commit()
data = { 'executable_id' : 1,
'testbed_id' : 2 }
response = self.client.post("/api/v1/deployments",
data=json.dumps(data),
content_type='application/json')
self.assertEquals(403, response.status_code)
self.assertEquals('Testbed is off-line, this process needs to be performed manually', response.json['message'])
# Verify that the executable was uploaded
data = { 'executable_id' : 1,
'testbed_id' : 3 }
response = self.client.post("/api/v1/deployments",
data=json.dumps(data),
content_type='application/json')
self.assertEquals(201, response.status_code)
self.assertEquals(1, response.json['executable_id'])
self.assertEquals(3, response.json['testbed_id'])
self.assertEquals(None, response.json['path'])
self.assertEquals(None, response.json['status'])
executable = db.session.query(Executable).filter_by(id=1).first()
testbed = db.session.query(Testbed).filter_by(id=3).first()
mock_upload_deployment.assert_called_with(executable, testbed)
def test_execute_application_type_torque_qsub(self, mock_shell,
mock_add_nodes):
"""
It verifies that the application type slurm sbatch is executed
"""
# First we verify that the testbed is of type TORQUE to be able
# to execute it, in this case it should give an error since it is
# not of type torque
# We define the different entities necessary for the test.
testbed = Testbed(
name="nova2",
on_line=True,
category="xxxx",
protocol="SSH",
endpoint="*****@*****.**",
package_formats=['sbatch', 'SINGULARITY'],
extra_config={
"enqueue_compss_sc_cfg":
"nova.cfg",
"enqueue_env_file":
"/home_nfs/home_ejarquej/installations/rc1707/COMPSs/compssenv"
})
db.session.add(testbed)
application = Application(name="super_app")
db.session.add(application)
db.session.commit() # So application and testbed get an id
executable = Executable()
executable.compilation_type = Executable.__type_torque_qsub__
executable.executable_file = "pepito.sh"
db.session.add(executable)
db.session.commit() # We do this so executable gets and id
deployment = Deployment()
deployment.testbed_id = testbed.id
deployment.executable_id = executable.id
db.session.add(
deployment) # We add the executable to the db so it has an id
execution_config = ExecutionConfiguration()
execution_config.execution_type = Executable.__type_torque_qsub__
execution_config.application = application
execution_config.testbed = testbed
execution_config.executable = executable
db.session.add(execution_config)
db.session.commit()
execution = Execution()
execution.execution_type = Executable.__type_torque_qsub__
execution.status = Execution.__status_submitted__
torque.execute_batch(execution, execution_config.id)
self.assertEquals(Execution.__status_failed__, execution.status)
self.assertEquals("Testbed does not support TORQUE:QSUB applications",
execution.output)
# If the testbed is off-line, execution isn't allowed also
testbed.category = Testbed.torque_category
testbed.on_line = False
db.session.commit()
execution = Execution()
execution.execution_type = Executable.__type_torque_qsub__
execution.status = Execution.__status_submitted__
torque.execute_batch(execution, execution_config.id)
self.assertEquals(Executable.__type_torque_qsub__,
execution.execution_type)
self.assertEquals(Execution.__status_failed__, execution.status)
self.assertEquals("Testbed is off-line", execution.output)
## Test executing
output = b'1208.cloudserver'
mock_shell.return_value = output
testbed.category = Testbed.torque_category
testbed.on_line = True
db.session.commit()
execution = Execution()
execution.execution_type = Executable.__type_torque_qsub__
execution.status = Execution.__status_submitted__
torque.execute_batch(execution, execution_config.id)
mock_shell.assert_called_with("qsub", "*****@*****.**",
["pepito.sh"])
execution = db.session.query(Execution).filter_by(
execution_configuration_id=execution_config.id).first()
self.assertEqual(execution.execution_type,
execution_config.execution_type)
self.assertEqual(execution.status, Execution.__status_running__)
self.assertEqual("1208.cloudserver", execution.batch_id)
def test_execute_srun(self, mock_shell):
"""
Verifyies the correct work of the function "execute_srun"
"""
# We define the different entities necessaryb for the test.
testbed = Testbed(
name="nova2",
on_line=True,
category="SLURM",
protocol="SSH",
endpoint="*****@*****.**",
package_formats=['sbatch', 'SINGULARITY'],
extra_config={
"enqueue_compss_sc_cfg":
"nova.cfg",
"enqueue_env_file":
"/home_nfs/home_ejarquej/installations/rc1707/COMPSs/compssenv"
})
application = Application(name="super_app")
executable = Executable()
executable.source_code_file = 'test.zip'
executable.compilation_script = 'gcc -X'
executable.compilation_type = "SINGULARITY:SRUN"
executable.singularity_app_folder = "/singularity/app/folder"
executable.singularity_image_file = "pepito.img"
executable.status = "COMPILED"
executable.application = application
deployment = Deployment()
deployment.testbed_id = testbed.id
deployment.executable_id = executable.id
deployment.path = "/pepito/pepito.img"
execution_config = ExecutionConfiguration()
execution_config.execution_type = "SINGULARITY:SRUN"
execution_config.application = application
execution_config.testbed = testbed
execution_config.executable = executable
execution_config.num_nodes = 2
#execution_config.num_gpus_per_node = 2
execution_config.num_cpus_per_node = 16
execution_config.exec_time = 10
execution_config.command = "/apps/application/master/Matmul 2 1024 12.34 /home_nfs/home_ejarquej/demo_test/cpu_gpu_run_data"
execution_config.compss_config = "--worker_in_master_cpus=12 --worker_in_master_memory=24000 --worker_working_dir=/home_nfs/home_ejarquej --lang=c --monitoring=1000 -d"
output = b' JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON)\n 4610 all singular garciad R 0:01 2 ns[55-56]\n'
mock_shell.return_value = output
# TEST starts here:
output_srun = slurm.execute_srun(testbed, execution_config, executable,
deployment, True)
call_1 = call('(', "*****@*****.**", [
"srun", "-N", "2", "-n", "16", "singularity", "run",
"/pepito/pepito.img", ">", "allout.txt", "2>&1", "&", ")", ";",
"sleep", "1;", "squeue"
])
# adding a new type of execution
execution_config = ExecutionConfiguration()
execution_config.execution_type = "SINGULARITY:SRUN"
execution_config.application = application
execution_config.testbed = testbed
execution_config.executable = executable
execution_config.num_gpus_per_node = 2
execution_config.num_cpus_per_node = 16
execution_config.exec_time = 10
execution_config.command = "/apps/application/master/Matmul 2 1024 12.34 /home_nfs/home_ejarquej/demo_test/cpu_gpu_run_data"
execution_config.compss_config = "--worker_in_master_cpus=12 --worker_in_master_memory=24000 --worker_working_dir=/home_nfs/home_ejarquej --lang=c --monitoring=1000 -d"
self.assertEquals(output, output_srun)
output_srun = slurm.execute_srun(testbed, execution_config, executable,
deployment, True)
call_2 = call('(', "*****@*****.**", [
"srun", "--gres=gpu:2", "-n", "16", "singularity", "run",
"/pepito/pepito.img", ">", "allout.txt", "2>&1", "&", ")", ";",
"sleep", "1;", "squeue"
])
calls = [call_1, call_2]
mock_shell.assert_has_calls(calls)
self.assertEquals(output, output_srun)