def test_parser_no_opt_job():
# this component has optimisation field
no_opt_component = "no-opt-batch-app"
# expected container for $no_opt_component
no_opt_expected_container_runtime = "docker://some-container"
# this component has optimisation field, but optimised container not found
job_script_component = "no-opt-batch-app-job-hpc"
test = TestConfig("no_opt_job", "test/opt_job_fixture.json")
parser.parse_data(test.parser_dest(), test.fixture())
service = test.service()
no_opt_component_template = service.get("topology_template").get(
"node_templates").get(no_opt_component)
image = no_opt_component_template.get("properties").get(
"container_runtime")
assert not "optimization" in no_opt_component_template
assert image == no_opt_expected_container_runtime
job_script_component_template = service.get("topology_template").get(
"node_templates").get(job_script_component)
content = job_script_component_template.get("properties").get(
"content", "")
assert len(content) > 0
assert "#PBS -N no-opt-skyline-extraction-training" in content
assert "#PBS -q ssd" in content
assert "#PBS -l nodes=1:gpus=1:ssd" in content
assert "#PBS -l procs=40" in content
def test_parser_opt():
# this component has optimisation field
opt_component = "optimization-skyline-extractor"
# expected container for $opt_component
opt_expected_container_runtime = "docker://modakopt/modak:tensorflow-2.1-gpu-src"
# this component has optimisation field, but optimised container not found
opt_not_found_component = "optimization-skyline-alignment"
# expected container for $opt_not_found_component
opt_not_found_expected_container_runtime = "snow-skyline-alignment:latest"
test = TestConfig("opt", "test/opt_fixture.json")
parser.parse_data(test.parser_dest(), test.fixture())
service = test.service()
opt_component_template = service.get("topology_template").get(
"node_templates").get(opt_component)
image_name = opt_component_template.get("properties").get("image_name")
assert not "optimization" in opt_component_template
assert image_name == opt_expected_container_runtime
opt_not_found_component_template = service.get("topology_template").get(
"node_templates").get(opt_not_found_component)
image_name = opt_not_found_component_template.get("properties").get(
"image_name")
assert not "optimization" in opt_not_found_component_template
assert image_name == opt_not_found_expected_container_runtime
def test_parser_tosca_capabilities():
test = TestConfig("capabilities", "test/mixed_tosca_types_fixture.json")
parser.parse_data(test.parser_dest(), test.fixture())
service = test.service()
assert service.get("capability_types") != None
assert service.get("capability_types").get(
"sodalite.capabilities.WM.JobResources") != None
assert service.get("capability_types").get(
"sodalite.capabilities.OptimisedTarget") != None
assert service.get("capability_types").get(
"sodalite.capabilities.OptimisedTarget").get("properties").get(
"target").get("type") == "string"
assert service.get("node_types").get(
"sodalite.capabilities.WM.JobResources") == None
assert service.get("node_types").get(
"sodalite.capabilities.OptimisedTarget") == None
assert service.get("topology_template").get("node_templates").get(
"hlrs-testbed").get("capabilities").get("optimisations").get(
"properties").get("target") == "optimised_target_name"
assert service.get("topology_template").get("node_templates").get(
"hlrs-testbed").get("capabilities").get("resources").get(
"properties").get("gpus") == 5
assert service.get("topology_template").get("node_templates").get(
"hlrs-testbed").get("capabilities").get("resources").get(
"properties").get("cpus") == 200
assert service.get("topology_template").get("node_templates").get(
"hlrs-testbed").get("capabilities").get("resources").get(
"properties").get("memory") == 650687
def test_parser_class_removal_from_tosca_types():
test = TestConfig("removed_tosca_types",
"test/mixed_tosca_types_fixture.json")
parser.parse_data(test.parser_dest(), test.fixture())
service = test.service()
assert service.get("capability_types").get(
"sodalite.capabilities.WM.JobResources").get("class") == None
assert service.get("capability_types").get(
"sodalite.capabilities.OptimisedTarget").get("class") == None
assert service.get("node_types").get("sodalite.nodes.hpc.WM").get(
"class") == None
def test_parser_opt_job(mocker):
mocker.patch('src.iacparser.ModakConfig.get_opt_image',
new=mock_modak_get_opt_image)
mocker.patch('src.iacparser.ModakConfig.get_opt_job_content',
new=mock_modak_get_opt_job_content)
# this component has optimisation field
opt_component = "batch-app"
# expected container for $opt_component
opt_expected_container_runtime = "docker://modakopt/modak:tensorflow-2.1-gpu-src"
# this component has optimisation field, but optimised container not found
job_script_component = "batch-app-job-hpc"
test = TestConfig("opt_job", "test/opt_job_fixture.json")
parser.parse_data(test.parser_dest(), test.fixture())
service = test.service()
opt_component_template = service.get("topology_template").get(
"node_templates").get(opt_component)
image = opt_component_template.get("properties").get("container_runtime")
assert not "optimization" in opt_component_template
assert image == opt_expected_container_runtime
job_script_component_template = service.get("topology_template").get(
"node_templates").get(job_script_component)
content = job_script_component_template.get("properties").get(
"content", "")
assert len(content) > 0
assert "#PBS -N skyline-extraction-training" in content
assert "#PBS -q ssd" in content
assert "#PBS -l nodes=1:gpus=1:ssd" in content
assert "#PBS -l procs=40" in content
def test_artifacts_extraction():
# expected ansible_files, expected ansible_paths, expected dependency_files, expected dependency_paths
expected = ([
'http://160.40.52.200:8084/Ansibles/ddce9b4f-7eef-46b9-b3d6-80f58eb5bb84',
'http://160.40.52.200:8084/Ansibles/3b95de8b-4815-4f00-9ee4-cedf49df4afb',
'http://160.40.52.200:8084/Ansibles/b8812f8a-7e1d-4acb-9258-45a5bdb45af2',
'http://160.40.52.200:8084/Ansibles/454ab4e5-43f2-479a-b31a-619596e0a696',
'http://160.40.52.200:8084/Ansibles/0ed0c21b-5614-48cf-82bc-cf7db4369f5b',
'http://160.40.52.200:8084/Ansibles/85767023-d35f-49bc-90c0-e79fe21351eb',
'http://160.40.52.200:8084/Ansibles/2a1e0e73-bcb8-4e6c-9c0d-0580cc5495b5',
'http://160.40.52.200:8084/Ansibles/85578876-343f-4e5b-ba1e-b1abfab8f6d3',
'http://160.40.52.200:8084/Ansibles/4250caa5-0a7d-42c1-98ae-92bb3d0dc82f',
'http://160.40.52.200:8084/Ansibles/7d35630c-fd17-45d6-9ccd-3c373ee0ee91',
'http://160.40.52.200:8084/Ansibles/15dbb35b-d70f-4458-a7c2-47fd39240fee',
'http://160.40.52.200:8084/Ansibles/d1e603eb-1261-4d73-90b9-683d3fd7d301',
'http://160.40.52.200:8084/Ansibles/67830dce-dabd-447f-81ab-9d2b57615331',
'http://160.40.52.200:8084/Ansibles/55f5be63-2220-4e62-9b6d-c15b69f82e9e',
'http://160.40.52.200:8084/Ansibles/88943b50-fdd9-4d5e-b72a-5fca96baaf27',
'http://160.40.52.200:8084/Ansibles/31f26aa0-cb6a-446c-b6c2-9c5e846b10e4',
'http://160.40.52.200:8084/Ansibles/0fc36efb-5153-49d5-9074-ed7fc66b88a4'
], [
'playbooks/ddce9b4f-7eef-46b9-b3d6-80f58eb5bb84_remove_network.yml',
'playbooks/3b95de8b-4815-4f00-9ee4-cedf49df4afb_add_network.yml',
'playbooks/b8812f8a-7e1d-4acb-9258-45a5bdb45af2_remove_container.yml',
'playbooks/454ab4e5-43f2-479a-b31a-619596e0a696_add_container.yml',
'playbooks/0ed0c21b-5614-48cf-82bc-cf7db4369f5b_remove_cert.yml',
'playbooks/85767023-d35f-49bc-90c0-e79fe21351eb_remove_volume.yml',
'playbooks/2a1e0e73-bcb8-4e6c-9c0d-0580cc5495b5_add_volume.yml',
'playbooks/85578876-343f-4e5b-ba1e-b1abfab8f6d3_create_docker_host.yml',
'playbooks/4250caa5-0a7d-42c1-98ae-92bb3d0dc82f_delete_docker_host.yml',
'playbooks/7d35630c-fd17-45d6-9ccd-3c373ee0ee91_login_user.yml',
'playbooks/15dbb35b-d70f-4458-a7c2-47fd39240fee_logout_user.yml',
'playbooks/d1e603eb-1261-4d73-90b9-683d3fd7d301_add_cert.yml',
'playbooks/67830dce-dabd-447f-81ab-9d2b57615331_security_rule_delete.yml',
'playbooks/55f5be63-2220-4e62-9b6d-c15b69f82e9e_security_rule_create.yml',
'playbooks/88943b50-fdd9-4d5e-b72a-5fca96baaf27_vm_create.yml',
'playbooks/31f26aa0-cb6a-446c-b6c2-9c5e846b10e4_vm_delete.yml',
'playbooks/0fc36efb-5153-49d5-9074-ed7fc66b88a4_configure_demo.yml'
], [
'http://160.40.52.200:8084/Ansibles/fde69a72-7375-4fef-adc7-e941cb985eec',
'http://160.40.52.200:8084/Ansibles/d6f34eaa-25cf-40fb-a31a-da244c0c9d37',
'http://160.40.52.200:8084/Ansibles/b8ad4903-01db-4433-8eaa-cbff67edee9b',
'http://160.40.52.200:8084/Ansibles/4ba88e9d-2f5a-43b7-b658-3ff59fe5c3cc',
'http://160.40.52.200:8084/Ansibles/c91b757e-2cb5-4596-b80f-bdbea6fb4ccd',
'http://160.40.52.200:8084/Ansibles/30c3abbe-7718-433e-bf3b-dab3093d690d',
'http://160.40.52.200:8084/Ansibles/2dd0cacf-014c-4c08-a211-d222e41d3c21'
], [
'artifacts/ca.key', 'artifacts/ca.crt',
'playbooks/egi_refresh_token.yml', 'playbooks/config.json.tmpl'
])
test = TestConfig("fixture", "test/fixture.json")
parsed_data = parser.parse_data(test.parser_dest(), test.fixture())
pds = []
exs = []
for i in range(0, 3):
pds.append(parsed_data[i])
exs.append(expected[i])
assert exs == pds
# check values reset
parsed_data = parser.parse_data(test.parser_dest(), test.fixture())
pds = []
exs = []
for i in range(0, 3):
pds.append(parsed_data[i])
exs.append(expected[i])
assert exs == pds
def test_fixture():
test = TestConfig('output_test')
parser.parse_data(test.parser_dest(), test.fixture())
return test