def test_ecs_unzip(tmp_path, settings, submission_file):
interface = ComponentInterfaceFactory(
kind=InterfaceKindChoices.ZIP, relative_path="preds.zip"
)
civ = ComponentInterfaceValueFactory(interface=interface)
with open(submission_file, "rb") as f:
civ.file.save("my_submission.zip", File(f))
settings.COMPONENTS_AMAZON_ECS_NFS_MOUNT_POINT = tmp_path
executor = AmazonECSExecutorStub(
job_id="algorithms-job-00000000-0000-0000-0000-000000000000",
exec_image_sha256="",
exec_image_repo_tag="",
memory_limit=4,
time_limit=60,
requires_gpu=False,
)
executor.provision(input_civs=[civ], input_prefixes={})
assert {str(f.relative_to(tmp_path)) for f in tmp_path.glob("**/*")} == {
"algorithms",
"algorithms/job",
"algorithms/job/00000000-0000-0000-0000-000000000000",
"algorithms/job/00000000-0000-0000-0000-000000000000/input",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/submission.csv",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/images",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/images/image10x10x10.mhd",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/images/image10x10x10.zraw",
"algorithms/job/00000000-0000-0000-0000-000000000000/output",
}
def test_input_prefixes(tmp_path, settings):
interfaces = [
ComponentInterfaceFactory(
kind=InterfaceKindChoices.BOOL, relative_path="test/bool.json"
),
ComponentInterfaceFactory(
kind=InterfaceKindChoices.IMAGE, relative_path="images/test-image"
),
ComponentInterfaceFactory(
kind=InterfaceKindChoices.CSV, relative_path="test.csv"
),
]
civs = [
ComponentInterfaceValueFactory(interface=interfaces[0], value=True),
ComponentInterfaceValueFactory(
interface=interfaces[1],
image=ImageFileFactory(
file__from_path=Path(__file__).parent.parent
/ "algorithms_tests"
/ "resources"
/ "input_file.tif"
).image,
),
ComponentInterfaceValueFactory(interface=interfaces[2]),
]
settings.COMPONENTS_AMAZON_ECS_NFS_MOUNT_POINT = tmp_path
executor = AmazonECSExecutorStub(
job_id="algorithms-job-00000000-0000-0000-0000-000000000000",
exec_image_sha256="",
exec_image_repo_tag="",
memory_limit=4,
time_limit=60,
requires_gpu=False,
)
executor.provision(
input_civs=civs,
input_prefixes={
str(civs[0].pk): "first/output/",
str(civs[1].pk): "second/output",
},
)
assert {str(f.relative_to(tmp_path)) for f in tmp_path.glob("**/*")} == {
"algorithms",
"algorithms/job",
"algorithms/job/00000000-0000-0000-0000-000000000000",
"algorithms/job/00000000-0000-0000-0000-000000000000/input",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/test.csv",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/first",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/first/output",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/first/output/test",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/first/output/test/bool.json",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/second",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/second/output",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/second/output/images",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/second/output/images/test-image",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/second/output/images/test-image/input_file.tif",
"algorithms/job/00000000-0000-0000-0000-000000000000/output",
}
def test_provision(tmp_path, settings):
interfaces = [
ComponentInterfaceFactory(
kind=InterfaceKindChoices.BOOL, relative_path="test/bool.json"
),
ComponentInterfaceFactory(
kind=InterfaceKindChoices.IMAGE, relative_path="images/test-image"
),
ComponentInterfaceFactory(
kind=InterfaceKindChoices.CSV, relative_path="test.csv"
),
]
civs = [
ComponentInterfaceValueFactory(interface=interfaces[0], value=True),
ComponentInterfaceValueFactory(
interface=interfaces[1],
image=ImageFileFactory(
file__from_path=Path(__file__).parent.parent
/ "algorithms_tests"
/ "resources"
/ "input_file.tif"
).image,
),
ComponentInterfaceValueFactory(interface=interfaces[2]),
]
civs[2].file.save("whatever.csv", ContentFile(b"foo,\nbar,\n"))
settings.COMPONENTS_AMAZON_ECS_NFS_MOUNT_POINT = tmp_path
executor = AmazonECSExecutorStub(
job_id="algorithms-job-00000000-0000-0000-0000-000000000000",
exec_image_sha256="",
exec_image_repo_tag="",
memory_limit=4,
time_limit=60,
requires_gpu=False,
)
executor.provision(input_civs=civs, input_prefixes={})
executor.execute()
executor.handle_event(
event={
# Minimal successful event
"taskDefinitionArn": "arn:aws:ecs:region:123456789012:task-definition/algorithms-job-00000000-0000-0000-0000-000000000000:1",
"group": "components-gpu",
"stopCode": "EssentialContainerExited",
"containers": [
{
"exitCode": 143,
"name": "algorithms-job-00000000-0000-0000-0000-000000000000-timeout",
},
{
"exitCode": 0,
"name": "algorithms-job-00000000-0000-0000-0000-000000000000",
},
],
"startedAt": "2021-09-25T10:50:24.248Z",
"stoppedAt": "2021-09-25T11:02:30.776Z",
}
)
assert executor.duration == datetime.timedelta(
seconds=726, microseconds=528000
)
assert {str(f.relative_to(tmp_path)) for f in tmp_path.glob("**/*")} == {
"algorithms",
"algorithms/job",
"algorithms/job/00000000-0000-0000-0000-000000000000",
"algorithms/job/00000000-0000-0000-0000-000000000000/input",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/test.csv",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/test",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/test/bool.json",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/images",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/images/test-image",
"algorithms/job/00000000-0000-0000-0000-000000000000/input/images/test-image/input_file.tif",
"algorithms/job/00000000-0000-0000-0000-000000000000/output",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/metrics.json",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/results.json",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/test.csv",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/test",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/test/bool.json",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/images",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/images/test-image",
"algorithms/job/00000000-0000-0000-0000-000000000000/output/images/test-image/input_file.tif",
}
# Exclude the CIV reading as this is unsupported
outputs = executor.get_outputs(output_interfaces=interfaces)
assert len(outputs) == 3
executor.deprovision()
assert {str(f.relative_to(tmp_path)) for f in tmp_path.glob("**/*")} == {
"algorithms",
"algorithms/job",
}