def test_execute_run_task_agent_token(monkeypatch):
boto3_client = MagicMock()
boto3_client.run_task.return_value = {}
monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client))
with set_temporary_config({"cloud.agent.auth_token": "test"}):
environment = FargateTaskEnvironment(cluster="test",
family="test",
taskDefinition="test")
environment.execute(
storage=Docker(registry_url="test",
image_name="image",
image_tag="tag"),
flow_location=".prefect/flows",
)
assert boto3_client.run_task.called
assert boto3_client.run_task.call_args[1]["taskDefinition"] == "test"
assert boto3_client.run_task.call_args[1]["overrides"] == {
"containerOverrides": [{
"name":
"flow-container",
"environment": [
{
"name": "PREFECT__CLOUD__AUTH_TOKEN",
"value": prefect.config.cloud.agent.get("auth_token"),
},
{
"name": "PREFECT__CONTEXT__FLOW_RUN_ID",
"value": "unknown"
},
{
"name": "PREFECT__CONTEXT__IMAGE",
"value": "test/image:tag"
},
{
"name": "PREFECT__CONTEXT__FLOW_FILE_PATH",
"value": ".prefect/flows",
},
],
}]
}
assert boto3_client.run_task.call_args[1]["launchType"] == "FARGATE"
assert boto3_client.run_task.call_args[1]["cluster"] == "test"
def test_create_fargate_task_environment():
environment = FargateTaskEnvironment()
assert environment
assert environment.executor_kwargs == {}
assert environment.labels == set()
assert environment.on_start is None
assert environment.on_exit is None
assert environment.metadata == {}
assert environment.logger.name == "prefect.FargateTaskEnvironment"
def test_create_fargate_task_environment_callbacks():
def f():
pass
environment = FargateTaskEnvironment(labels=["foo"], on_start=f, on_exit=f)
assert environment
assert environment.labels == set(["foo"])
assert environment.on_start is f
assert environment.on_exit is f
def test_run_flow_calls_callbacks(monkeypatch):
start_func = MagicMock()
exit_func = MagicMock()
environment = FargateTaskEnvironment(on_start=start_func, on_exit=exit_func)
flow_runner = MagicMock()
monkeypatch.setattr(
"prefect.engine.get_default_flow_runner_class",
MagicMock(return_value=flow_runner),
)
with tempfile.TemporaryDirectory() as directory:
d = Local(directory)
d.add_flow(prefect.Flow("name"))
gql_return = MagicMock(
return_value=MagicMock(
data=MagicMock(
flow_run=[
GraphQLResult(
{
"flow": GraphQLResult(
{"name": "name", "storage": d.serialize(),}
)
}
)
],
)
)
)
client = MagicMock()
client.return_value.graphql = gql_return
monkeypatch.setattr("prefect.environments.execution.base.Client", client)
with set_temporary_config({"cloud.auth_token": "test"}), prefect.context(
{"flow_run_id": "id"}
):
environment.run_flow()
assert flow_runner.call_args[1]["flow"].name == "name"
assert start_func.called
assert exit_func.called
def test_create_fargate_task_environment_with_deprecated_executor_kwargs():
with set_temporary_config({
"engine.executor.default_class":
"prefect.engine.executors.LocalDaskExecutor"
}):
with pytest.warns(UserWarning, match="executor_kwargs"):
environment = FargateTaskEnvironment(
executor_kwargs={"scheduler": "synchronous"})
assert isinstance(environment.executor, LocalDaskExecutor)
assert environment.executor.scheduler == "synchronous"
def test_create_fargate_task_environment_aws_creds_environment(monkeypatch):
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "id")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "secret")
monkeypatch.setenv("REGION_NAME", "region")
environment = FargateTaskEnvironment(labels=["foo"])
assert environment
assert environment.labels == set(["foo"])
assert environment.aws_access_key_id == "id"
assert environment.aws_secret_access_key == "secret"
assert environment.region_name == "region"
def test_create_fargate_task_environment_aws_creds_provided():
environment = FargateTaskEnvironment(
labels=["foo"],
aws_access_key_id="id",
aws_secret_access_key="secret",
region_name="region",
)
assert environment
assert environment.labels == set(["foo"])
assert environment.aws_access_key_id == "id"
assert environment.aws_secret_access_key == "secret"
assert environment.region_name == "region"
def test_roundtrip_cloudpickle():
with tempfile.TemporaryDirectory() as directory:
with open(os.path.join(directory, "job.yaml"), "w+") as file:
file.write("job")
environment = FargateTaskEnvironment(cluster="test")
assert environment.task_run_kwargs == {"cluster": "test"}
new = cloudpickle.loads(cloudpickle.dumps(environment))
assert isinstance(new, FargateTaskEnvironment)
assert new.task_run_kwargs == {"cluster": "test"}
def test_parse_task_run_kwargs():
environment = FargateTaskEnvironment()
kwarg_dict = {
"cluster": "test",
"taskDefinition": "test",
"count": "test",
"startedBy": "test",
"group": "test",
"placementConstraints": "test",
"placementStrategy": "test",
"platformVersion": "test",
"networkConfiguration": "test",
"tags": "test",
"enableECSManagedTags": "test",
"propagateTags": "test",
}
task_definition_kwargs, task_run_kwargs = environment._parse_kwargs(kwarg_dict)
assert task_run_kwargs == kwarg_dict
assert task_definition_kwargs == {"placementConstraints": "test", "tags": "test"}
def test_environment_run():
class MyExecutor(LocalDaskExecutor):
submit_called = False
def submit(self, *args, **kwargs):
self.submit_called = True
return super().submit(*args, **kwargs)
global_dict = {}
@prefect.task
def add_to_dict():
global_dict["run"] = True
executor = MyExecutor()
environment = FargateTaskEnvironment(executor=executor)
flow = prefect.Flow("test", tasks=[add_to_dict], environment=environment)
environment.run(flow=flow)
assert global_dict.get("run") is True
assert executor.submit_called
def test_run_flow(monkeypatch):
environment = FargateTaskEnvironment()
flow_runner = MagicMock()
monkeypatch.setattr(
"prefect.engine.get_default_flow_runner_class",
MagicMock(return_value=flow_runner),
)
with tempfile.TemporaryDirectory() as directory:
with open(os.path.join(directory, "flow_env.prefect"), "w+"):
flow = prefect.Flow("test")
flow_path = os.path.join(directory, "flow_env.prefect")
with open(flow_path, "wb") as f:
cloudpickle.dump(flow, f)
with set_temporary_config({"cloud.auth_token": "test"}):
with prefect.context(flow_file_path=os.path.join(
directory, "flow_env.prefect")):
environment.run_flow()
assert flow_runner.call_args[1]["flow"].name == "test"
def test_create_fargate_task_environment_labels():
environment = FargateTaskEnvironment(labels=["foo"])
assert environment
assert environment.labels == set(["foo"])
def test_create_fargate_task_environment_with_executor_kwargs():
environment = FargateTaskEnvironment(executor_kwargs={"test": "here"})
assert environment
assert environment.executor_kwargs == {"test": "here"}
def test_parse_task_definition_and_run_kwargs():
environment = FargateTaskEnvironment()
def_kwarg_dict = {
"family": "test",
"taskRoleArn": "test",
"executionRoleArn": "test",
"networkMode": "test",
"containerDefinitions": "test",
"volumes": "test",
"placementConstraints": "test",
"requiresCompatibilities": "test",
"cpu": "test",
"memory": "test",
"tags": "test",
"pidMode": "test",
"ipcMode": "test",
"proxyConfiguration": "test",
"inferenceAccelerators": "test",
}
run_kwarg_dict = {
"cluster": "test",
"taskDefinition": "test",
"count": "test",
"startedBy": "test",
"group": "test",
"placementConstraints": "test",
"placementStrategy": "test",
"platformVersion": "test",
"networkConfiguration": "test",
"tags": "test",
"enableECSManagedTags": "test",
"propagateTags": "test",
}
kwarg_dict = {
"family": "test",
"taskRoleArn": "test",
"executionRoleArn": "test",
"networkMode": "test",
"containerDefinitions": "test",
"volumes": "test",
"placementConstraints": "test",
"requiresCompatibilities": "test",
"cpu": "test",
"memory": "test",
"tags": "test",
"pidMode": "test",
"ipcMode": "test",
"proxyConfiguration": "test",
"inferenceAccelerators": "test",
"cluster": "test",
"taskDefinition": "test",
"count": "test",
"startedBy": "test",
"group": "test",
"placementStrategy": "test",
"platformVersion": "test",
"networkConfiguration": "test",
"enableECSManagedTags": "test",
"propagateTags": "test",
}
task_definition_kwargs, task_run_kwargs = environment._parse_kwargs(kwarg_dict)
assert task_definition_kwargs == def_kwarg_dict
assert task_run_kwargs == run_kwarg_dict
def test_setup_definition_register(monkeypatch):
boto3_client = MagicMock()
boto3_client.describe_task_definition.side_effect = ClientError({}, None)
boto3_client.register_task_definition.return_value = {}
monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client))
environment = FargateTaskEnvironment(
family="test",
containerDefinitions=[{
"name": "flow-container",
"image": "image",
"command": [],
"environment": [],
"essential": True,
}],
)
environment.setup(
Flow(
"test",
storage=Docker(registry_url="test",
image_name="image",
image_tag="tag"),
))
assert boto3_client.describe_task_definition.called
assert boto3_client.register_task_definition.called
assert boto3_client.register_task_definition.call_args[1][
"family"] == "test"
assert boto3_client.register_task_definition.call_args[1][
"containerDefinitions"] == [{
"name":
"flow-container",
"image":
"test/image:tag",
"command": [
"/bin/sh",
"-c",
"python -c 'import prefect; prefect.environments.execution.load_and_run_flow()'",
],
"environment": [
{
"name": "PREFECT__CLOUD__GRAPHQL",
"value": prefect.config.cloud.graphql,
},
{
"name": "PREFECT__CLOUD__USE_LOCAL_SECRETS",
"value": "false"
},
{
"name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS",
"value": "prefect.engine.cloud.CloudFlowRunner",
},
{
"name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS",
"value": "prefect.engine.cloud.CloudTaskRunner",
},
{
"name": "PREFECT__LOGGING__LOG_TO_CLOUD",
"value": "true"
},
{
"name": "PREFECT__LOGGING__EXTRA_LOGGERS",
"value": "[]",
},
],
"essential":
True,
}]
def register_workflow(prefect_register_token_secret_name: str):
"""
Registers the workflow to Prefect Cloud
Parameters:
prefect_register_token_secret_name [str]
-- name of aws secrets manager secret where prefect register token is stored
"""
flow_module = __import__("flow")
flow_name = f"{env}_{flow_module.flow.name}"
flow_module.flow.name = flow_name
flow_module.flow.environment = FargateTaskEnvironment(
requiresCompatibilities=["FARGATE"],
region=aws_region,
labels=[f"{env}_dataflow_automation"],
taskDefinition=flow_name,
family=flow_name,
cpu="512",
memory="3072",
networkMode="awsvpc",
networkConfiguration={
"awsvpcConfiguration": {
"assignPublicIp": "ENABLED",
"subnets": subnets,
"securityGroups": [],
}
},
containerDefinitions=[{
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-region": aws_region,
"awslogs-group": f"{env}_dataflow_automation_workflows",
"awslogs-stream-prefix": flow_name,
},
}
}],
executionRoleArn=execution_role_arn,
taskRoleArn=task_role_arn,
cluster=f"{env}_dataflow_automation_workflows",
)
# Set the flow storage. Where to get the code from
flow_module.flow.storage = Docker(
registry_url=f"{account_id}.dkr.ecr.{aws_region}.amazonaws.com",
image_name=flow_name,
image_tag="latest",
python_dependencies=["boto3"],
env_vars={"PYTHONPATH": "/opt/prefect/flows"},
)
# Authenticate to ECR as the registration process pushes the image to AWS
ecr_authenticate()
# Instantiate the prefect client
prefect_client = Client(api_token=get_prefect_token(
secret_name=prefect_register_token_secret_name))
# Create ECR repository
create_ecr_repository(flow_name=flow_name)
# Register the Workflow
prefect_client.register(flow=flow_module.flow,
project_name=f"{env}_dataflow_automation")
def test_entire_environment_process_together(monkeypatch):
boto3_client = MagicMock()
boto3_client.describe_task_definition.side_effect = ClientError({}, None)
boto3_client.register_task_definition.return_value = {}
boto3_client.run_task.return_value = {}
monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client))
flow_runner = MagicMock()
monkeypatch.setattr(
"prefect.engine.get_default_flow_runner_class",
MagicMock(return_value=flow_runner),
)
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "id")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "secret")
monkeypatch.setenv("AWS_SESSION_TOKEN", "session")
monkeypatch.setenv("REGION_NAME", "region")
with prefect.context({"flow_run_id": "id"}), set_temporary_config(
{"cloud.auth_token": "test", "logging.extra_loggers": "['test_logger']",}
):
storage = Docker(registry_url="test", image_name="image", image_tag="tag")
flow = Flow("name", storage=storage)
environment = FargateTaskEnvironment(
containerDefinitions=[
{
"name": "flow-container",
"image": "image",
"command": [],
"environment": [],
"essential": True,
}
],
cluster="test",
family="test",
taskDefinition="test",
)
assert environment
assert environment.aws_access_key_id == "id"
assert environment.aws_secret_access_key == "secret"
assert environment.aws_session_token == "session"
assert environment.region_name == "region"
environment.setup(flow=flow)
assert boto3_client.describe_task_definition.called
assert boto3_client.register_task_definition.called
assert boto3_client.register_task_definition.call_args[1]["family"] == "test"
assert boto3_client.register_task_definition.call_args[1][
"containerDefinitions"
] == [
{
"name": "flow-container",
"image": "test/image:tag",
"command": [
"/bin/sh",
"-c",
"python -c 'import prefect; prefect.environments.FargateTaskEnvironment().run_flow()'",
],
"environment": [
{
"name": "PREFECT__CLOUD__GRAPHQL",
"value": prefect.config.cloud.graphql,
},
{"name": "PREFECT__CLOUD__USE_LOCAL_SECRETS", "value": "false"},
{
"name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS",
"value": "prefect.engine.cloud.CloudFlowRunner",
},
{
"name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS",
"value": "prefect.engine.cloud.CloudTaskRunner",
},
{"name": "PREFECT__LOGGING__LOG_TO_CLOUD", "value": "true"},
{
"name": "PREFECT__LOGGING__EXTRA_LOGGERS",
"value": "['test_logger']",
},
],
"essential": True,
}
]
environment.execute(flow=flow)
assert boto3_client.run_task.called
assert boto3_client.run_task.call_args[1]["taskDefinition"] == "test"
assert boto3_client.run_task.call_args[1]["overrides"] == {
"containerOverrides": [
{
"name": "flow-container",
"environment": [
{
"name": "PREFECT__CLOUD__AUTH_TOKEN",
"value": prefect.config.cloud.get("auth_token"),
},
{"name": "PREFECT__CONTEXT__FLOW_RUN_ID", "value": "id"},
{"name": "PREFECT__CONTEXT__IMAGE", "value": "test/image:tag"},
],
}
]
}
assert boto3_client.run_task.call_args[1]["launchType"] == "FARGATE"
assert boto3_client.run_task.call_args[1]["cluster"] == "test"
with tempfile.TemporaryDirectory() as directory:
d = Local(directory)
d.add_flow(prefect.Flow("name"))
gql_return = MagicMock(
return_value=MagicMock(
data=MagicMock(
flow_run=[
GraphQLResult(
{
"flow": GraphQLResult(
{"name": "name", "storage": d.serialize(),}
)
}
)
],
)
)
)
client = MagicMock()
client.return_value.graphql = gql_return
monkeypatch.setattr("prefect.environments.execution.base.Client", client)
with set_temporary_config({"cloud.auth_token": "test"}):
environment.run_flow()
assert flow_runner.call_args[1]["flow"].name == "name"
def test_create_fargate_task_environment_with_executor():
executor = LocalDaskExecutor()
environment = FargateTaskEnvironment(executor=executor)
assert environment.executor is executor
@task
def extract():
"""Get a list of data"""
return [1, 2, 3]
@task
def transform(data):
"""Multiply the input by 10"""
return [i * 10 for i in data]
@task
def load(data):
"""Print the data to indicate it was received"""
print("Here's your data: {}".format(data))
from prefect import Flow
with Flow("ETL-fte",
environment=FargateTaskEnvironment(),
storage=Docker(base_image="prefecthq/prefect:latest")) as flow:
e = extract()
t = transform(e)
l = load(t)
flow.storage.add_flow(flow)
flow.storage.build()
# flow.deploy(project_name="Demo", registry_url="joshmeek18", image_name="flows")
def test_entire_environment_process_together(monkeypatch):
boto3_client = MagicMock()
boto3_client.describe_task_definition.side_effect = ClientError({}, None)
boto3_client.register_task_definition.return_value = {}
boto3_client.run_task.return_value = {}
monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client))
flow_runner = MagicMock()
monkeypatch.setattr(
"prefect.engine.get_default_flow_runner_class",
MagicMock(return_value=flow_runner),
)
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "id")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "secret")
monkeypatch.setenv("AWS_SESSION_TOKEN", "session")
monkeypatch.setenv("REGION_NAME", "region")
with prefect.context({"flow_run_id": "id"}):
storage = Docker(registry_url="test",
image_name="image",
image_tag="tag")
environment = FargateTaskEnvironment(
containerDefinitions=[{
"name": "flow-container",
"image": "image",
"command": [],
"environment": [],
"essential": True,
}],
cluster="test",
family="test",
taskDefinition="test",
)
assert environment
assert environment.aws_access_key_id == "id"
assert environment.aws_secret_access_key == "secret"
assert environment.aws_session_token == "session"
assert environment.region_name == "region"
environment.setup(storage=storage)
assert boto3_client.describe_task_definition.called
assert boto3_client.register_task_definition.called
assert boto3_client.register_task_definition.call_args[1][
"family"] == "test"
assert boto3_client.register_task_definition.call_args[1][
"containerDefinitions"] == [{
"name":
"flow-container",
"image":
"test/image:tag",
"command": [
"/bin/sh",
"-c",
"python -c 'import prefect; prefect.Flow.load(prefect.context.flow_file_path).environment.run_flow()'",
],
"environment": [
{
"name": "PREFECT__CLOUD__GRAPHQL",
"value": prefect.config.cloud.graphql,
},
{
"name": "PREFECT__CLOUD__USE_LOCAL_SECRETS",
"value": "false"
},
{
"name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS",
"value": "prefect.engine.cloud.CloudFlowRunner",
},
{
"name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS",
"value": "prefect.engine.cloud.CloudTaskRunner",
},
{
"name": "PREFECT__LOGGING__LOG_TO_CLOUD",
"value": "true"
},
],
"essential":
True,
}]
environment.execute(storage=storage, flow_location=".prefect/flows")
assert boto3_client.run_task.called
assert boto3_client.run_task.call_args[1]["taskDefinition"] == "test"
assert boto3_client.run_task.call_args[1]["overrides"] == {
"containerOverrides": [{
"name":
"flow-container",
"environment": [
{
"name": "PREFECT__CLOUD__AUTH_TOKEN",
"value": prefect.config.cloud.agent.auth_token,
},
{
"name": "PREFECT__CONTEXT__FLOW_RUN_ID",
"value": "id"
},
{
"name": "PREFECT__CONTEXT__IMAGE",
"value": "test/image:tag"
},
{
"name": "PREFECT__CONTEXT__FLOW_FILE_PATH",
"value": ".prefect/flows",
},
],
}]
}
assert boto3_client.run_task.call_args[1]["launchType"] == "FARGATE"
assert boto3_client.run_task.call_args[1]["cluster"] == "test"
with tempfile.TemporaryDirectory() as directory:
with open(os.path.join(directory, "flow_env.prefect"), "w+"):
flow = prefect.Flow("test")
flow_path = os.path.join(directory, "flow_env.prefect")
with open(flow_path, "wb") as f:
cloudpickle.dump(flow, f)
with set_temporary_config({"cloud.auth_token": "test"}):
with prefect.context(flow_file_path=os.path.join(
directory, "flow_env.prefect")):
environment.run_flow()
assert flow_runner.call_args[1]["flow"].name == "test"
def test_fargate_task_environment_dependencies():
environment = FargateTaskEnvironment()
assert environment.dependencies == ["boto3", "botocore"]
def test_validate_definition_not_changed_when_out_of_order_in_second_container(
monkeypatch, ):
existing_task_definition = {
"containerDefinitions": [
{
"environment": [
{
"name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS",
"value": "prefect.engine.cloud.CloudFlowRunner",
},
{
"name": "PREFECT__CLOUD__USE_LOCAL_SECRETS",
"value": "false"
},
{
"name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS",
"value": "prefect.engine.cloud.CloudTaskRunner",
},
{
"name": "PREFECT__LOGGING__LOG_TO_CLOUD",
"value": "true"
},
{
"name": "PREFECT__LOGGING__EXTRA_LOGGERS",
"value": str(config.logging.extra_loggers),
},
# This is added first in _render_task_definition_kwargs, so it's at the end now
{
"name": "PREFECT__CLOUD__GRAPHQL",
"value": config.cloud.graphql
},
],
"name":
"flow-container",
"image":
"test/image:tag",
"command": [
"/bin/sh",
"-c",
"python -c 'import prefect; prefect.environments.execution.load_and_run_flow()'",
],
},
{
"environment": [
{
"name": "foo",
"value": "bar",
},
{
"name": "foo2",
"value": "bar2",
},
{
"name": "PREFECT__CLOUD__GRAPHQL",
"value": config.cloud.graphql
},
{
"name": "PREFECT__CLOUD__USE_LOCAL_SECRETS",
"value": "false"
},
{
"name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS",
"value": "prefect.engine.cloud.CloudFlowRunner",
},
{
"name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS",
"value": "prefect.engine.cloud.CloudTaskRunner",
},
{
"name": "PREFECT__LOGGING__LOG_TO_CLOUD",
"value": "true"
},
{
"name": "PREFECT__LOGGING__EXTRA_LOGGERS",
"value": str(config.logging.extra_loggers),
},
],
"secrets": [
{
"name": "1",
"valueFrom": "1"
},
{
"name": "2",
"valueFrom": "2"
},
],
"mountPoints": [
{
"sourceVolume": "1",
"containerPath": "1",
"readOnly": False
},
{
"sourceVolume": "2",
"containerPath": "2",
"readOnly": False
},
],
"extraHosts": [
{
"hostname": "1",
"ipAddress": "1"
},
{
"hostname": "2",
"ipAddress": "2"
},
],
"volumesFrom": [
{
"sourceContainer": "1",
"readOnly": False
},
{
"sourceContainer": "2",
"readOnly": False
},
],
"ulimits": [
{
"name": "cpu",
"softLimit": 1,
"hardLimit": 1
},
{
"name": "memlock",
"softLimit": 2,
"hardLimit": 2
},
],
"portMappings": [
{
"containerPort": 80,
"hostPort": 80,
"protocol": "tcp"
},
{
"containerPort": 81,
"hostPort": 81,
"protocol": "tcp"
},
],
"logConfiguration": {
"logDriver":
"awslogs",
"options": {},
"secretOptions": [
{
"name": "1",
"valueFrom": "1"
},
{
"name": "2",
"valueFrom": "2"
},
],
},
"name":
"some-other-container",
"image":
"test/image:tag",
"command": [
"/bin/sh",
],
},
],
"memory":
256,
"cpu":
512,
}
boto3_client = MagicMock()
boto3_client.describe_task_definition.return_value = {
"taskDefinition": existing_task_definition
}
monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client))
environment = FargateTaskEnvironment(
memory=256,
cpu=512,
containerDefinitions=[
{},
{
"environment": [
{
"name": "foo2",
"value": "bar2",
},
{
"name": "foo",
"value": "bar",
},
],
"secrets": [
{
"name": "2",
"valueFrom": "2"
},
{
"name": "1",
"valueFrom": "1"
},
],
"mountPoints": [
{
"sourceVolume": "2",
"containerPath": "2",
"readOnly": False
},
{
"sourceVolume": "1",
"containerPath": "1",
"readOnly": False
},
],
"extraHosts": [
{
"hostname": "2",
"ipAddress": "2"
},
{
"hostname": "1",
"ipAddress": "1"
},
],
"volumesFrom": [
{
"sourceContainer": "2",
"readOnly": False
},
{
"sourceContainer": "1",
"readOnly": False
},
],
"ulimits": [
{
"name": "memlock",
"softLimit": 2,
"hardLimit": 2
},
{
"name": "cpu",
"softLimit": 1,
"hardLimit": 1
},
],
"portMappings": [
{
"containerPort": 81,
"hostPort": 81,
"protocol": "tcp"
},
{
"containerPort": 80,
"hostPort": 80,
"protocol": "tcp"
},
],
"logConfiguration": {
"logDriver":
"awslogs",
"options": {},
"secretOptions": [
{
"name": "2",
"valueFrom": "2"
},
{
"name": "1",
"valueFrom": "1"
},
],
},
"name":
"some-other-container",
"image":
"test/image:tag",
"command": [
"/bin/sh",
],
},
],
)
environment.setup(
Flow(
"test",
storage=Docker(registry_url="test",
image_name="image",
image_tag="tag"),
))
assert boto3_client.describe_task_definition.called
assert not boto3_client.register_task_definition.called
def test_validate_definition_not_changed_when_names_are_in_arn(monkeypatch):
existing_task_definition = {
"containerDefinitions": [{
"environment": [
{
"name": "PREFECT__CLOUD__GRAPHQL",
"value": config.cloud.graphql
},
{
"name": "PREFECT__CLOUD__USE_LOCAL_SECRETS",
"value": "false"
},
{
"name": "PREFECT__ENGINE__FLOW_RUNNER__DEFAULT_CLASS",
"value": "prefect.engine.cloud.CloudFlowRunner",
},
{
"name": "PREFECT__ENGINE__TASK_RUNNER__DEFAULT_CLASS",
"value": "prefect.engine.cloud.CloudTaskRunner",
},
{
"name": "PREFECT__LOGGING__LOG_TO_CLOUD",
"value": "true"
},
{
"name": "PREFECT__LOGGING__EXTRA_LOGGERS",
"value": str(config.logging.extra_loggers),
},
],
"name":
"flow-container",
"image":
"test/image:tag",
"command": [
"/bin/sh",
"-c",
"python -c 'import prefect; prefect.environments.execution.load_and_run_flow()'",
],
}],
"taskRoleArn":
"arn:aws:iam::000000000000:role/my-role-name",
"memory":
256,
"cpu":
512,
}
boto3_client = MagicMock()
boto3_client.describe_task_definition.return_value = {
"taskDefinition": existing_task_definition
}
monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client))
environment = FargateTaskEnvironment(memory=256,
cpu=512,
taskRoleArn="my-role-name")
environment.setup(
Flow(
"test",
storage=Docker(registry_url="test",
image_name="image",
image_tag="tag"),
))
assert boto3_client.describe_task_definition.called
assert not boto3_client.register_task_definition.called
flow = Flow(
"Fargate Task Environment",
environment=FargateTaskEnvironment(
aws_session_token="MY_AWS_SESSION_TOKEN",
region="us-east-1",
cpu="256",
memory="512",
networkConfiguration={
"awsvpcConfiguration": {
"assignPublicIp": "ENABLED",
"subnets": ["MY_SUBNET_ID"],
"securityGroups": ["MY_SECURITY_GROUP"],
}
},
family="my_flow",
taskRoleArn="MY_TASK_ROLE_ARN",
executionRoleArn="MY_EXECUTION_ROLE_ARN",
containerDefinitions={
"name": "flow-container",
"image": "image",
"command": [],
"environment": [],
"essential": True,
}
),
storage=Docker(registry_url="joshmeek18", image_name="flows"),
)
# set task dependencies using imperative API
