def test_handle_stopped_cancelled_task(): executor = AmazonECSExecutorStub( job_id="algorithms-job-00000000-0000-0000-0000-000000000000", exec_image_sha256="", exec_image_repo_tag="", memory_limit=4, time_limit=60, requires_gpu=False, ) event = { "taskDefinitionArn": "arn:aws:ecs:region:123456789012:task-definition/algorithms-job-00000000-0000-0000-0000-000000000000:1", "group": "components-gpu", "stopCode": "UserInitiated", "containers": [ { # No container exit in this case "name": "algorithms-job-00000000-0000-0000-0000-000000000000-timeout" }, { # No container exit in this case "name": "algorithms-job-00000000-0000-0000-0000-000000000000" }, ], "createdAt": "2021-09-25T10:50:24.248Z", # No startedAt in this case "stoppedAt": "2021-09-25T11:02:30.776Z", } with pytest.raises(TaskCancelled): # Task should be retried executor._get_container_exit_codes(event=event)
def test_handle_stopped_successful_fast_task(): executor = AmazonECSExecutorStub( job_id="algorithms-job-00000000-0000-0000-0000-000000000000", exec_image_sha256="", exec_image_repo_tag="", memory_limit=4, time_limit=60, requires_gpu=False, ) event = { "taskDefinitionArn": "arn:aws:ecs:region:123456789012:task-definition/algorithms-job-00000000-0000-0000-0000-000000000000:1", "group": "components-gpu", "stopCode": "TaskFailedToStart", # as not all container had time to start "containers": [ { # No container exit in this case "name": "algorithms-job-00000000-0000-0000-0000-000000000000-timeout" }, { "exitCode": 0, "name": "algorithms-job-00000000-0000-0000-0000-000000000000", }, ], "createdAt": "2021-09-25T10:50:24.248Z", # No startedAt in this case "stoppedAt": "2021-09-25T11:02:30.776Z", } assert executor._get_container_exit_codes(event=event) == { "algorithms-job-00000000-0000-0000-0000-000000000000": 0 }
def test_ecs_unzip(tmp_path, settings, submission_file): interface = ComponentInterfaceFactory( kind=InterfaceKindChoices.ZIP, relative_path="preds.zip" ) civ = ComponentInterfaceValueFactory(interface=interface) with open(submission_file, "rb") as f: civ.file.save("my_submission.zip", File(f)) settings.COMPONENTS_AMAZON_ECS_NFS_MOUNT_POINT = tmp_path executor = AmazonECSExecutorStub( job_id="algorithms-job-00000000-0000-0000-0000-000000000000", exec_image_sha256="", exec_image_repo_tag="", memory_limit=4, time_limit=60, requires_gpu=False, ) executor.provision(input_civs=[civ], input_prefixes={}) assert {str(f.relative_to(tmp_path)) for f in tmp_path.glob("**/*")} == { "algorithms", "algorithms/job", "algorithms/job/00000000-0000-0000-0000-000000000000", "algorithms/job/00000000-0000-0000-0000-000000000000/input", "algorithms/job/00000000-0000-0000-0000-000000000000/input/submission.csv", "algorithms/job/00000000-0000-0000-0000-000000000000/input/images", "algorithms/job/00000000-0000-0000-0000-000000000000/input/images/image10x10x10.mhd", "algorithms/job/00000000-0000-0000-0000-000000000000/input/images/image10x10x10.zraw", "algorithms/job/00000000-0000-0000-0000-000000000000/output", }
def test_input_prefixes(tmp_path, settings): interfaces = [ ComponentInterfaceFactory( kind=InterfaceKindChoices.BOOL, relative_path="test/bool.json" ), ComponentInterfaceFactory( kind=InterfaceKindChoices.IMAGE, relative_path="images/test-image" ), ComponentInterfaceFactory( kind=InterfaceKindChoices.CSV, relative_path="test.csv" ), ] civs = [ ComponentInterfaceValueFactory(interface=interfaces[0], value=True), ComponentInterfaceValueFactory( interface=interfaces[1], image=ImageFileFactory( file__from_path=Path(__file__).parent.parent / "algorithms_tests" / "resources" / "input_file.tif" ).image, ), ComponentInterfaceValueFactory(interface=interfaces[2]), ] settings.COMPONENTS_AMAZON_ECS_NFS_MOUNT_POINT = tmp_path executor = AmazonECSExecutorStub( job_id="algorithms-job-00000000-0000-0000-0000-000000000000", exec_image_sha256="", exec_image_repo_tag="", memory_limit=4, time_limit=60, requires_gpu=False, ) executor.provision( input_civs=civs, input_prefixes={ str(civs[0].pk): "first/output/", str(civs[1].pk): "second/output", }, ) assert {str(f.relative_to(tmp_path)) for f in tmp_path.glob("**/*")} == { "algorithms", "algorithms/job", "algorithms/job/00000000-0000-0000-0000-000000000000", "algorithms/job/00000000-0000-0000-0000-000000000000/input", "algorithms/job/00000000-0000-0000-0000-000000000000/input/test.csv", "algorithms/job/00000000-0000-0000-0000-000000000000/input/first", "algorithms/job/00000000-0000-0000-0000-000000000000/input/first/output", "algorithms/job/00000000-0000-0000-0000-000000000000/input/first/output/test", "algorithms/job/00000000-0000-0000-0000-000000000000/input/first/output/test/bool.json", "algorithms/job/00000000-0000-0000-0000-000000000000/input/second", "algorithms/job/00000000-0000-0000-0000-000000000000/input/second/output", "algorithms/job/00000000-0000-0000-0000-000000000000/input/second/output/images", "algorithms/job/00000000-0000-0000-0000-000000000000/input/second/output/images/test-image", "algorithms/job/00000000-0000-0000-0000-000000000000/input/second/output/images/test-image/input_file.tif", "algorithms/job/00000000-0000-0000-0000-000000000000/output", }
def test_set_duration_fast_task(): executor = AmazonECSExecutorStub( job_id="algorithms-job-00000000-0000-0000-0000-000000000000", exec_image_sha256="", exec_image_repo_tag="", exec_image_file=None, memory_limit=4, requires_gpu=False, ) executor._set_duration( event={ "createdAt": "2021-09-25T10:50:24.248Z", "stoppedAt": "2021-09-25T11:02:30.776Z", }) assert executor.duration == datetime.timedelta(seconds=726, microseconds=528000)
def test_handle_stopped_successful_task(): executor = AmazonECSExecutorStub( job_id="algorithms-job-00000000-0000-0000-0000-000000000000", exec_image_sha256="", exec_image_repo_tag="", exec_image_file=None, memory_limit=4, requires_gpu=False, ) event = { "taskDefinitionArn": "arn:aws:ecs:region:123456789012:task-definition/algorithms-job-00000000-0000-0000-0000-000000000000:1", "group": "components-gpu", "stopCode": "EssentialContainerExited", "containers": [ { "exitCode": 143, "name": "algorithms-job-00000000-0000-0000-0000-000000000000-timeout", }, { "exitCode": 0, "name": "algorithms-job-00000000-0000-0000-0000-000000000000", }, ], "startedAt": "2021-09-25T10:50:24.248Z", "stoppedAt": "2021-09-25T11:02:30.776Z", } assert executor._get_container_exit_codes(event=event) == { "algorithms-job-00000000-0000-0000-0000-000000000000": 0, "algorithms-job-00000000-0000-0000-0000-000000000000-timeout": 143, }
def test_provision(tmp_path, settings): interfaces = [ ComponentInterfaceFactory( kind=InterfaceKindChoices.BOOL, relative_path="test/bool.json" ), ComponentInterfaceFactory( kind=InterfaceKindChoices.IMAGE, relative_path="images/test-image" ), ComponentInterfaceFactory( kind=InterfaceKindChoices.CSV, relative_path="test.csv" ), ] civs = [ ComponentInterfaceValueFactory(interface=interfaces[0], value=True), ComponentInterfaceValueFactory( interface=interfaces[1], image=ImageFileFactory( file__from_path=Path(__file__).parent.parent / "algorithms_tests" / "resources" / "input_file.tif" ).image, ), ComponentInterfaceValueFactory(interface=interfaces[2]), ] civs[2].file.save("whatever.csv", ContentFile(b"foo,\nbar,\n")) settings.COMPONENTS_AMAZON_ECS_NFS_MOUNT_POINT = tmp_path executor = AmazonECSExecutorStub( job_id="algorithms-job-00000000-0000-0000-0000-000000000000", exec_image_sha256="", exec_image_repo_tag="", memory_limit=4, time_limit=60, requires_gpu=False, ) executor.provision(input_civs=civs, input_prefixes={}) executor.execute() executor.handle_event( event={ # Minimal successful event "taskDefinitionArn": "arn:aws:ecs:region:123456789012:task-definition/algorithms-job-00000000-0000-0000-0000-000000000000:1", "group": "components-gpu", "stopCode": "EssentialContainerExited", "containers": [ { "exitCode": 143, "name": "algorithms-job-00000000-0000-0000-0000-000000000000-timeout", }, { "exitCode": 0, "name": "algorithms-job-00000000-0000-0000-0000-000000000000", }, ], "startedAt": "2021-09-25T10:50:24.248Z", "stoppedAt": "2021-09-25T11:02:30.776Z", } ) assert executor.duration == datetime.timedelta( seconds=726, microseconds=528000 ) assert {str(f.relative_to(tmp_path)) for f in tmp_path.glob("**/*")} == { "algorithms", "algorithms/job", "algorithms/job/00000000-0000-0000-0000-000000000000", "algorithms/job/00000000-0000-0000-0000-000000000000/input", "algorithms/job/00000000-0000-0000-0000-000000000000/input/test.csv", "algorithms/job/00000000-0000-0000-0000-000000000000/input/test", "algorithms/job/00000000-0000-0000-0000-000000000000/input/test/bool.json", "algorithms/job/00000000-0000-0000-0000-000000000000/input/images", "algorithms/job/00000000-0000-0000-0000-000000000000/input/images/test-image", "algorithms/job/00000000-0000-0000-0000-000000000000/input/images/test-image/input_file.tif", "algorithms/job/00000000-0000-0000-0000-000000000000/output", "algorithms/job/00000000-0000-0000-0000-000000000000/output/metrics.json", "algorithms/job/00000000-0000-0000-0000-000000000000/output/results.json", "algorithms/job/00000000-0000-0000-0000-000000000000/output/test.csv", "algorithms/job/00000000-0000-0000-0000-000000000000/output/test", "algorithms/job/00000000-0000-0000-0000-000000000000/output/test/bool.json", "algorithms/job/00000000-0000-0000-0000-000000000000/output/images", "algorithms/job/00000000-0000-0000-0000-000000000000/output/images/test-image", "algorithms/job/00000000-0000-0000-0000-000000000000/output/images/test-image/input_file.tif", } # Exclude the CIV reading as this is unsupported outputs = executor.get_outputs(output_interfaces=interfaces) assert len(outputs) == 3 executor.deprovision() assert {str(f.relative_to(tmp_path)) for f in tmp_path.glob("**/*")} == { "algorithms", "algorithms/job", }