def test_execute_run_task_agent_token(monkeypatch): boto3_client = MagicMock() boto3_client.run_task.return_value = {} monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client)) with set_temporary_config({"cloud.agent.auth_token": "test"}): environment = FargateTaskEnvironment(cluster="test", family="test", taskDefinition="test") environment.execute( storage=Docker(registry_url="test", image_name="image", image_tag="tag"), flow_location=".prefect/flows", ) assert boto3_client.run_task.called assert boto3_client.run_task.call_args[1]["taskDefinition"] == "test" assert boto3_client.run_task.call_args[1]["overrides"] == { "containerOverrides": [{ "name": "flow-container", "environment": [ { "name": "PREFECT__CLOUD__AUTH_TOKEN", "value": prefect.config.cloud.agent.get("auth_token"), }, { "name": "PREFECT__CONTEXT__FLOW_RUN_ID", "value": "unknown" }, { "name": "PREFECT__CONTEXT__IMAGE", "value": "test/image:tag" }, { "name": "PREFECT__CONTEXT__FLOW_FILE_PATH", "value": ".prefect/flows", }, ], }] } assert boto3_client.run_task.call_args[1]["launchType"] == "FARGATE" assert boto3_client.run_task.call_args[1]["cluster"] == "test"
def test_create_fargate_task_environment(): environment = FargateTaskEnvironment() assert environment assert environment.executor_kwargs == {} assert environment.labels == set() assert environment.on_start is None assert environment.on_exit is None assert environment.metadata == {} assert environment.logger.name == "prefect.FargateTaskEnvironment"
def test_create_fargate_task_environment_callbacks(): def f(): pass environment = FargateTaskEnvironment(labels=["foo"], on_start=f, on_exit=f) assert environment assert environment.labels == set(["foo"]) assert environment.on_start is f assert environment.on_exit is f
def test_run_flow_calls_callbacks(monkeypatch): start_func = MagicMock() exit_func = MagicMock() environment = FargateTaskEnvironment(on_start=start_func, on_exit=exit_func) flow_runner = MagicMock() monkeypatch.setattr( "prefect.engine.get_default_flow_runner_class", MagicMock(return_value=flow_runner), ) with tempfile.TemporaryDirectory() as directory: d = Local(directory) d.add_flow(prefect.Flow("name")) gql_return = MagicMock( return_value=MagicMock( data=MagicMock( flow_run=[ GraphQLResult( { "flow": GraphQLResult( {"name": "name", "storage": d.serialize(),} ) } ) ], ) ) ) client = MagicMock() client.return_value.graphql = gql_return monkeypatch.setattr("prefect.environments.execution.base.Client", client) with set_temporary_config({"cloud.auth_token": "test"}), prefect.context( {"flow_run_id": "id"} ): environment.run_flow() assert flow_runner.call_args[1]["flow"].name == "name" assert start_func.called assert exit_func.called
def test_create_fargate_task_environment_with_deprecated_executor_kwargs(): with set_temporary_config({ "engine.executor.default_class": "prefect.engine.executors.LocalDaskExecutor" }): with pytest.warns(UserWarning, match="executor_kwargs"): environment = FargateTaskEnvironment( executor_kwargs={"scheduler": "synchronous"}) assert isinstance(environment.executor, LocalDaskExecutor) assert environment.executor.scheduler == "synchronous"
def test_create_fargate_task_environment_aws_creds_environment(monkeypatch): monkeypatch.setenv("AWS_ACCESS_KEY_ID", "id") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "secret") monkeypatch.setenv("REGION_NAME", "region") environment = FargateTaskEnvironment(labels=["foo"]) assert environment assert environment.labels == set(["foo"]) assert environment.aws_access_key_id == "id" assert environment.aws_secret_access_key == "secret" assert environment.region_name == "region"
def test_create_fargate_task_environment_aws_creds_provided(): environment = FargateTaskEnvironment( labels=["foo"], aws_access_key_id="id", aws_secret_access_key="secret", region_name="region", ) assert environment assert environment.labels == set(["foo"]) assert environment.aws_access_key_id == "id" assert environment.aws_secret_access_key == "secret" assert environment.region_name == "region"
def test_roundtrip_cloudpickle(): with tempfile.TemporaryDirectory() as directory: with open(os.path.join(directory, "job.yaml"), "w+") as file: file.write("job") environment = FargateTaskEnvironment(cluster="test") assert environment.task_run_kwargs == {"cluster": "test"} new = cloudpickle.loads(cloudpickle.dumps(environment)) assert isinstance(new, FargateTaskEnvironment) assert new.task_run_kwargs == {"cluster": "test"}
def test_parse_task_run_kwargs(): environment = FargateTaskEnvironment() kwarg_dict = { "cluster": "test", "taskDefinition": "test", "count": "test", "startedBy": "test", "group": "test", "placementConstraints": "test", "placementStrategy": "test", "platformVersion": "test", "networkConfiguration": "test", "tags": "test", "enableECSManagedTags": "test", "propagateTags": "test", } task_definition_kwargs, task_run_kwargs = environment._parse_kwargs(kwarg_dict) assert task_run_kwargs == kwarg_dict assert task_definition_kwargs == {"placementConstraints": "test", "tags": "test"}
def test_environment_run(): class MyExecutor(LocalDaskExecutor): submit_called = False def submit(self, *args, **kwargs): self.submit_called = True return super().submit(*args, **kwargs) global_dict = {} @prefect.task def add_to_dict(): global_dict["run"] = True executor = MyExecutor() environment = FargateTaskEnvironment(executor=executor) flow = prefect.Flow("test", tasks=[add_to_dict], environment=environment) environment.run(flow=flow) assert global_dict.get("run") is True assert executor.submit_called
def test_run_flow(monkeypatch): environment = FargateTaskEnvironment() flow_runner = MagicMock() monkeypatch.setattr( "prefect.engine.get_default_flow_runner_class", MagicMock(return_value=flow_runner), ) with tempfile.TemporaryDirectory() as directory: with open(os.path.join(directory, "flow_env.prefect"), "w+"): flow = prefect.Flow("test") flow_path = os.path.join(directory, "flow_env.prefect") with open(flow_path, "wb") as f: cloudpickle.dump(flow, f) with set_temporary_config({"cloud.auth_token": "test"}): with prefect.context(flow_file_path=os.path.join( directory, "flow_env.prefect")): environment.run_flow() assert flow_runner.call_args[1]["flow"].name == "test"
def test_create_fargate_task_environment_labels(): environment = FargateTaskEnvironment(labels=["foo"]) assert environment assert environment.labels == set(["foo"])
def test_create_fargate_task_environment_with_executor_kwargs(): environment = FargateTaskEnvironment(executor_kwargs={"test": "here"}) assert environment assert environment.executor_kwargs == {"test": "here"}
def test_parse_task_definition_and_run_kwargs(): environment = FargateTaskEnvironment() def_kwarg_dict = { "family": "test", "taskRoleArn": "test", "executionRoleArn": "test", "networkMode": "test", "containerDefinitions": "test", "volumes": "test", "placementConstraints": "test", "requiresCompatibilities": "test", "cpu": "test", "memory": "test", "tags": "test", "pidMode": "test", "ipcMode": "test", "proxyConfiguration": "test", "inferenceAccelerators": "test", } run_kwarg_dict = { "cluster": "test", "taskDefinition": "test", "count": "test", "startedBy": "test", "group": "test", "placementConstraints": "test", "placementStrategy": "test", "platformVersion": "test", "networkConfiguration": "test", "tags": "test", "enableECSManagedTags": "test", "propagateTags": "test", } kwarg_dict = { "family": "test", "taskRoleArn": "test", "executionRoleArn": "test", "networkMode": "test", "containerDefinitions": "test", "volumes": "test", "placementConstraints": "test", "requiresCompatibilities": "test", "cpu": "test", "memory": "test", "tags": "test", "pidMode": "test", "ipcMode": "test", "proxyConfiguration": "test", "inferenceAccelerators": "test", "cluster": "test", "taskDefinition": "test", "count": "test", "startedBy": "test", "group": "test", "placementStrategy": "test", "platformVersion": "test", "networkConfiguration": "test", "enableECSManagedTags": "test", "propagateTags": "test", } task_definition_kwargs, task_run_kwargs = environment._parse_kwargs(kwarg_dict) assert task_definition_kwargs == def_kwarg_dict assert task_run_kwargs == run_kwarg_dict
def test_setup_definition_register(monkeypatch): boto3_client = MagicMock() boto3_client.describe_task_definition.side_effect = ClientError({}, None) boto3_client.register_task_definition.return_value = {} monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client)) environment = FargateTaskEnvironment( family="test", containerDefinitions=[{ "name": "flow-container", "image": "image", "command": [], "environment": [], "essential": True, }], ) environment.setup( Flow( "test", storage=Docker(registry_url="test", image_name="image", image_tag="tag"), )) assert boto3_client.describe_task_definition.called assert boto3_client.register_task_definition.called assert boto3_client.register_task_definition.call_args[1][ "family"] == "test" assert boto3_client.register_task_definition.call_args[1][ "containerDefinitions"] == [{ "name": "flow-container", "image": "test/image:tag", "command": [ "/bin/sh", "-c", "python -c 'import prefect; prefect.environments.execution.load_and_run_flow()'", ], "environment": [ { "name": "PREFECT__CLOUD__GRAPHQL", "value": prefect.config.cloud.graphql, }, { "name": "PREFECT__CLOUD__USE_LOCAL_SECRETS", "value": "false" }, { "name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudFlowRunner", }, { "name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudTaskRunner", }, { "name": "PREFECT__LOGGING__LOG_TO_CLOUD", "value": "true" }, { "name": "PREFECT__LOGGING__EXTRA_LOGGERS", "value": "[]", }, ], "essential": True, }]
def register_workflow(prefect_register_token_secret_name: str): """ Registers the workflow to Prefect Cloud Parameters: prefect_register_token_secret_name [str] -- name of aws secrets manager secret where prefect register token is stored """ flow_module = __import__("flow") flow_name = f"{env}_{flow_module.flow.name}" flow_module.flow.name = flow_name flow_module.flow.environment = FargateTaskEnvironment( requiresCompatibilities=["FARGATE"], region=aws_region, labels=[f"{env}_dataflow_automation"], taskDefinition=flow_name, family=flow_name, cpu="512", memory="3072", networkMode="awsvpc", networkConfiguration={ "awsvpcConfiguration": { "assignPublicIp": "ENABLED", "subnets": subnets, "securityGroups": [], } }, containerDefinitions=[{ "logConfiguration": { "logDriver": "awslogs", "options": { "awslogs-region": aws_region, "awslogs-group": f"{env}_dataflow_automation_workflows", "awslogs-stream-prefix": flow_name, }, } }], executionRoleArn=execution_role_arn, taskRoleArn=task_role_arn, cluster=f"{env}_dataflow_automation_workflows", ) # Set the flow storage. Where to get the code from flow_module.flow.storage = Docker( registry_url=f"{account_id}.dkr.ecr.{aws_region}.amazonaws.com", image_name=flow_name, image_tag="latest", python_dependencies=["boto3"], env_vars={"PYTHONPATH": "/opt/prefect/flows"}, ) # Authenticate to ECR as the registration process pushes the image to AWS ecr_authenticate() # Instantiate the prefect client prefect_client = Client(api_token=get_prefect_token( secret_name=prefect_register_token_secret_name)) # Create ECR repository create_ecr_repository(flow_name=flow_name) # Register the Workflow prefect_client.register(flow=flow_module.flow, project_name=f"{env}_dataflow_automation")
def test_entire_environment_process_together(monkeypatch): boto3_client = MagicMock() boto3_client.describe_task_definition.side_effect = ClientError({}, None) boto3_client.register_task_definition.return_value = {} boto3_client.run_task.return_value = {} monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client)) flow_runner = MagicMock() monkeypatch.setattr( "prefect.engine.get_default_flow_runner_class", MagicMock(return_value=flow_runner), ) monkeypatch.setenv("AWS_ACCESS_KEY_ID", "id") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "secret") monkeypatch.setenv("AWS_SESSION_TOKEN", "session") monkeypatch.setenv("REGION_NAME", "region") with prefect.context({"flow_run_id": "id"}), set_temporary_config( {"cloud.auth_token": "test", "logging.extra_loggers": "['test_logger']",} ): storage = Docker(registry_url="test", image_name="image", image_tag="tag") flow = Flow("name", storage=storage) environment = FargateTaskEnvironment( containerDefinitions=[ { "name": "flow-container", "image": "image", "command": [], "environment": [], "essential": True, } ], cluster="test", family="test", taskDefinition="test", ) assert environment assert environment.aws_access_key_id == "id" assert environment.aws_secret_access_key == "secret" assert environment.aws_session_token == "session" assert environment.region_name == "region" environment.setup(flow=flow) assert boto3_client.describe_task_definition.called assert boto3_client.register_task_definition.called assert boto3_client.register_task_definition.call_args[1]["family"] == "test" assert boto3_client.register_task_definition.call_args[1][ "containerDefinitions" ] == [ { "name": "flow-container", "image": "test/image:tag", "command": [ "/bin/sh", "-c", "python -c 'import prefect; prefect.environments.FargateTaskEnvironment().run_flow()'", ], "environment": [ { "name": "PREFECT__CLOUD__GRAPHQL", "value": prefect.config.cloud.graphql, }, {"name": "PREFECT__CLOUD__USE_LOCAL_SECRETS", "value": "false"}, { "name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudFlowRunner", }, { "name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudTaskRunner", }, {"name": "PREFECT__LOGGING__LOG_TO_CLOUD", "value": "true"}, { "name": "PREFECT__LOGGING__EXTRA_LOGGERS", "value": "['test_logger']", }, ], "essential": True, } ] environment.execute(flow=flow) assert boto3_client.run_task.called assert boto3_client.run_task.call_args[1]["taskDefinition"] == "test" assert boto3_client.run_task.call_args[1]["overrides"] == { "containerOverrides": [ { "name": "flow-container", "environment": [ { "name": "PREFECT__CLOUD__AUTH_TOKEN", "value": prefect.config.cloud.get("auth_token"), }, {"name": "PREFECT__CONTEXT__FLOW_RUN_ID", "value": "id"}, {"name": "PREFECT__CONTEXT__IMAGE", "value": "test/image:tag"}, ], } ] } assert boto3_client.run_task.call_args[1]["launchType"] == "FARGATE" assert boto3_client.run_task.call_args[1]["cluster"] == "test" with tempfile.TemporaryDirectory() as directory: d = Local(directory) d.add_flow(prefect.Flow("name")) gql_return = MagicMock( return_value=MagicMock( data=MagicMock( flow_run=[ GraphQLResult( { "flow": GraphQLResult( {"name": "name", "storage": d.serialize(),} ) } ) ], ) ) ) client = MagicMock() client.return_value.graphql = gql_return monkeypatch.setattr("prefect.environments.execution.base.Client", client) with set_temporary_config({"cloud.auth_token": "test"}): environment.run_flow() assert flow_runner.call_args[1]["flow"].name == "name"
def test_create_fargate_task_environment_with_executor(): executor = LocalDaskExecutor() environment = FargateTaskEnvironment(executor=executor) assert environment.executor is executor
@task def extract(): """Get a list of data""" return [1, 2, 3] @task def transform(data): """Multiply the input by 10""" return [i * 10 for i in data] @task def load(data): """Print the data to indicate it was received""" print("Here's your data: {}".format(data)) from prefect import Flow with Flow("ETL-fte", environment=FargateTaskEnvironment(), storage=Docker(base_image="prefecthq/prefect:latest")) as flow: e = extract() t = transform(e) l = load(t) flow.storage.add_flow(flow) flow.storage.build() # flow.deploy(project_name="Demo", registry_url="joshmeek18", image_name="flows")
def test_entire_environment_process_together(monkeypatch): boto3_client = MagicMock() boto3_client.describe_task_definition.side_effect = ClientError({}, None) boto3_client.register_task_definition.return_value = {} boto3_client.run_task.return_value = {} monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client)) flow_runner = MagicMock() monkeypatch.setattr( "prefect.engine.get_default_flow_runner_class", MagicMock(return_value=flow_runner), ) monkeypatch.setenv("AWS_ACCESS_KEY_ID", "id") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "secret") monkeypatch.setenv("AWS_SESSION_TOKEN", "session") monkeypatch.setenv("REGION_NAME", "region") with prefect.context({"flow_run_id": "id"}): storage = Docker(registry_url="test", image_name="image", image_tag="tag") environment = FargateTaskEnvironment( containerDefinitions=[{ "name": "flow-container", "image": "image", "command": [], "environment": [], "essential": True, }], cluster="test", family="test", taskDefinition="test", ) assert environment assert environment.aws_access_key_id == "id" assert environment.aws_secret_access_key == "secret" assert environment.aws_session_token == "session" assert environment.region_name == "region" environment.setup(storage=storage) assert boto3_client.describe_task_definition.called assert boto3_client.register_task_definition.called assert boto3_client.register_task_definition.call_args[1][ "family"] == "test" assert boto3_client.register_task_definition.call_args[1][ "containerDefinitions"] == [{ "name": "flow-container", "image": "test/image:tag", "command": [ "/bin/sh", "-c", "python -c 'import prefect; prefect.Flow.load(prefect.context.flow_file_path).environment.run_flow()'", ], "environment": [ { "name": "PREFECT__CLOUD__GRAPHQL", "value": prefect.config.cloud.graphql, }, { "name": "PREFECT__CLOUD__USE_LOCAL_SECRETS", "value": "false" }, { "name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudFlowRunner", }, { "name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudTaskRunner", }, { "name": "PREFECT__LOGGING__LOG_TO_CLOUD", "value": "true" }, ], "essential": True, }] environment.execute(storage=storage, flow_location=".prefect/flows") assert boto3_client.run_task.called assert boto3_client.run_task.call_args[1]["taskDefinition"] == "test" assert boto3_client.run_task.call_args[1]["overrides"] == { "containerOverrides": [{ "name": "flow-container", "environment": [ { "name": "PREFECT__CLOUD__AUTH_TOKEN", "value": prefect.config.cloud.agent.auth_token, }, { "name": "PREFECT__CONTEXT__FLOW_RUN_ID", "value": "id" }, { "name": "PREFECT__CONTEXT__IMAGE", "value": "test/image:tag" }, { "name": "PREFECT__CONTEXT__FLOW_FILE_PATH", "value": ".prefect/flows", }, ], }] } assert boto3_client.run_task.call_args[1]["launchType"] == "FARGATE" assert boto3_client.run_task.call_args[1]["cluster"] == "test" with tempfile.TemporaryDirectory() as directory: with open(os.path.join(directory, "flow_env.prefect"), "w+"): flow = prefect.Flow("test") flow_path = os.path.join(directory, "flow_env.prefect") with open(flow_path, "wb") as f: cloudpickle.dump(flow, f) with set_temporary_config({"cloud.auth_token": "test"}): with prefect.context(flow_file_path=os.path.join( directory, "flow_env.prefect")): environment.run_flow() assert flow_runner.call_args[1]["flow"].name == "test"
def test_fargate_task_environment_dependencies(): environment = FargateTaskEnvironment() assert environment.dependencies == ["boto3", "botocore"]
def test_validate_definition_not_changed_when_out_of_order_in_second_container( monkeypatch, ): existing_task_definition = { "containerDefinitions": [ { "environment": [ { "name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudFlowRunner", }, { "name": "PREFECT__CLOUD__USE_LOCAL_SECRETS", "value": "false" }, { "name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudTaskRunner", }, { "name": "PREFECT__LOGGING__LOG_TO_CLOUD", "value": "true" }, { "name": "PREFECT__LOGGING__EXTRA_LOGGERS", "value": str(config.logging.extra_loggers), }, # This is added first in _render_task_definition_kwargs, so it's at the end now { "name": "PREFECT__CLOUD__GRAPHQL", "value": config.cloud.graphql }, ], "name": "flow-container", "image": "test/image:tag", "command": [ "/bin/sh", "-c", "python -c 'import prefect; prefect.environments.execution.load_and_run_flow()'", ], }, { "environment": [ { "name": "foo", "value": "bar", }, { "name": "foo2", "value": "bar2", }, { "name": "PREFECT__CLOUD__GRAPHQL", "value": config.cloud.graphql }, { "name": "PREFECT__CLOUD__USE_LOCAL_SECRETS", "value": "false" }, { "name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudFlowRunner", }, { "name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudTaskRunner", }, { "name": "PREFECT__LOGGING__LOG_TO_CLOUD", "value": "true" }, { "name": "PREFECT__LOGGING__EXTRA_LOGGERS", "value": str(config.logging.extra_loggers), }, ], "secrets": [ { "name": "1", "valueFrom": "1" }, { "name": "2", "valueFrom": "2" }, ], "mountPoints": [ { "sourceVolume": "1", "containerPath": "1", "readOnly": False }, { "sourceVolume": "2", "containerPath": "2", "readOnly": False }, ], "extraHosts": [ { "hostname": "1", "ipAddress": "1" }, { "hostname": "2", "ipAddress": "2" }, ], "volumesFrom": [ { "sourceContainer": "1", "readOnly": False }, { "sourceContainer": "2", "readOnly": False }, ], "ulimits": [ { "name": "cpu", "softLimit": 1, "hardLimit": 1 }, { "name": "memlock", "softLimit": 2, "hardLimit": 2 }, ], "portMappings": [ { "containerPort": 80, "hostPort": 80, "protocol": "tcp" }, { "containerPort": 81, "hostPort": 81, "protocol": "tcp" }, ], "logConfiguration": { "logDriver": "awslogs", "options": {}, "secretOptions": [ { "name": "1", "valueFrom": "1" }, { "name": "2", "valueFrom": "2" }, ], }, "name": "some-other-container", "image": "test/image:tag", "command": [ "/bin/sh", ], }, ], "memory": 256, "cpu": 512, } boto3_client = MagicMock() boto3_client.describe_task_definition.return_value = { "taskDefinition": existing_task_definition } monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client)) environment = FargateTaskEnvironment( memory=256, cpu=512, containerDefinitions=[ {}, { "environment": [ { "name": "foo2", "value": "bar2", }, { "name": "foo", "value": "bar", }, ], "secrets": [ { "name": "2", "valueFrom": "2" }, { "name": "1", "valueFrom": "1" }, ], "mountPoints": [ { "sourceVolume": "2", "containerPath": "2", "readOnly": False }, { "sourceVolume": "1", "containerPath": "1", "readOnly": False }, ], "extraHosts": [ { "hostname": "2", "ipAddress": "2" }, { "hostname": "1", "ipAddress": "1" }, ], "volumesFrom": [ { "sourceContainer": "2", "readOnly": False }, { "sourceContainer": "1", "readOnly": False }, ], "ulimits": [ { "name": "memlock", "softLimit": 2, "hardLimit": 2 }, { "name": "cpu", "softLimit": 1, "hardLimit": 1 }, ], "portMappings": [ { "containerPort": 81, "hostPort": 81, "protocol": "tcp" }, { "containerPort": 80, "hostPort": 80, "protocol": "tcp" }, ], "logConfiguration": { "logDriver": "awslogs", "options": {}, "secretOptions": [ { "name": "2", "valueFrom": "2" }, { "name": "1", "valueFrom": "1" }, ], }, "name": "some-other-container", "image": "test/image:tag", "command": [ "/bin/sh", ], }, ], ) environment.setup( Flow( "test", storage=Docker(registry_url="test", image_name="image", image_tag="tag"), )) assert boto3_client.describe_task_definition.called assert not boto3_client.register_task_definition.called
def test_validate_definition_not_changed_when_names_are_in_arn(monkeypatch): existing_task_definition = { "containerDefinitions": [{ "environment": [ { "name": "PREFECT__CLOUD__GRAPHQL", "value": config.cloud.graphql }, { "name": "PREFECT__CLOUD__USE_LOCAL_SECRETS", "value": "false" }, { "name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudFlowRunner", }, { "name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS", "value": "prefect.engine.cloud.CloudTaskRunner", }, { "name": "PREFECT__LOGGING__LOG_TO_CLOUD", "value": "true" }, { "name": "PREFECT__LOGGING__EXTRA_LOGGERS", "value": str(config.logging.extra_loggers), }, ], "name": "flow-container", "image": "test/image:tag", "command": [ "/bin/sh", "-c", "python -c 'import prefect; prefect.environments.execution.load_and_run_flow()'", ], }], "taskRoleArn": "arn:aws:iam::000000000000:role/my-role-name", "memory": 256, "cpu": 512, } boto3_client = MagicMock() boto3_client.describe_task_definition.return_value = { "taskDefinition": existing_task_definition } monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client)) environment = FargateTaskEnvironment(memory=256, cpu=512, taskRoleArn="my-role-name") environment.setup( Flow( "test", storage=Docker(registry_url="test", image_name="image", image_tag="tag"), )) assert boto3_client.describe_task_definition.called assert not boto3_client.register_task_definition.called
flow = Flow( "Fargate Task Environment", environment=FargateTaskEnvironment( aws_session_token="MY_AWS_SESSION_TOKEN", region="us-east-1", cpu="256", memory="512", networkConfiguration={ "awsvpcConfiguration": { "assignPublicIp": "ENABLED", "subnets": ["MY_SUBNET_ID"], "securityGroups": ["MY_SECURITY_GROUP"], } }, family="my_flow", taskRoleArn="MY_TASK_ROLE_ARN", executionRoleArn="MY_EXECUTION_ROLE_ARN", containerDefinitions={ "name": "flow-container", "image": "image", "command": [], "environment": [], "essential": True, } ), storage=Docker(registry_url="joshmeek18", image_name="flows"), ) # set task dependencies using imperative API