Exemplo n.º 1
0
def test_k8s_agent_includes_agent_labels_in_job(monkeypatch, runner_token):
    k8s_config = MagicMock()
    monkeypatch.setattr("kubernetes.config", k8s_config)

    flow_run = GraphQLResult({
        "flow":
        GraphQLResult({
            "storage":
            Docker(registry_url="test", image_name="name",
                   image_tag="tag").serialize(),
            "id":
            "id",
        }),
        "id":
        "id",
    })

    agent = KubernetesAgent(labels=["foo", "bar"])
    job = agent.replace_job_spec_yaml(flow_run)
    env = job["spec"]["template"]["spec"]["containers"][0]["env"]
    assert env[4]["value"] == "['foo', 'bar']"
def test_setup_doesnt_pass_if_private_registry(monkeypatch):
    environment = DaskKubernetesEnvironment(private_registry=True)
    assert environment.docker_secret == "DOCKER_REGISTRY_CREDENTIALS"

    config = MagicMock()
    monkeypatch.setattr("kubernetes.config", config)

    v1 = MagicMock()
    v1.list_namespaced_secret.return_value = MagicMock(items=[])
    monkeypatch.setattr("kubernetes.client",
                        MagicMock(CoreV1Api=MagicMock(return_value=v1)))

    create_secret = MagicMock()
    monkeypatch.setattr(
        "prefect.environments.DaskKubernetesEnvironment._create_namespaced_secret",
        create_secret,
    )
    with set_temporary_config({"cloud.auth_token": "test"}):
        environment.setup(storage=Docker())

    assert create_secret.called
Exemplo n.º 3
0
def test_k8s_agent_replace_yaml_no_pull_secrets(monkeypatch, runner_token):
    k8s_config = MagicMock()
    monkeypatch.setattr("kubernetes.config", k8s_config)

    flow_run = GraphQLResult({
        "flow":
        GraphQLResult({
            "storage":
            Docker(registry_url="test", image_name="name",
                   image_tag="tag").serialize(),
            "id":
            "id",
        }),
        "id":
        "id",
    })

    agent = KubernetesAgent()
    job = agent.replace_job_spec_yaml(flow_run)

    assert not job["spec"]["template"]["spec"]["imagePullSecrets"][0]["name"]
Exemplo n.º 4
0
def test_run_healthchecks_arg(ignore_healthchecks):

    with tempfile.TemporaryDirectory() as tempdir_outside:

        with open(os.path.join(tempdir_outside, "test"), "w+") as t:
            t.write("asdf")

        with tempfile.TemporaryDirectory() as tempdir:
            storage = Docker(ignore_healthchecks=ignore_healthchecks)

            f = Flow("test")
            storage.add_flow(f)
            dpath = storage.create_dockerfile_object(directory=tempdir)

            with open(dpath, "r") as dockerfile:
                output = dockerfile.read()

            if ignore_healthchecks:
                assert "RUN python /opt/prefect/healthcheck.py" not in output
            else:
                assert "RUN python /opt/prefect/healthcheck.py" in output
Exemplo n.º 5
0
def test_empty_docker_storage_on_tagged_commit(monkeypatch, platform, url,
                                               no_docker_host_var):
    monkeypatch.setattr("prefect.environments.storage.docker.sys.platform",
                        platform)
    monkeypatch.setattr(sys, "version_info", MagicMock(major=3, minor=6))
    monkeypatch.setattr(prefect, "__version__", "0.9.2")

    storage = Docker()

    assert not storage.registry_url
    assert storage.base_image == "prefecthq/prefect:0.9.2-python3.6"
    assert not storage.image_name
    assert not storage.image_tag
    assert storage.python_dependencies == ["wheel"]
    assert storage.env_vars == {
        "PREFECT__USER_CONFIG_PATH": "/opt/prefect/config.toml"
    }
    assert not storage.files
    assert storage.prefect_version
    assert storage.base_url == url
    assert not storage.local_image
Exemplo n.º 6
0
def test_empty_docker_storage(monkeypatch, platform, url):
    monkeypatch.setattr("prefect.environments.storage.docker.sys.platform",
                        platform)
    monkeypatch.setattr(sys, "version_info", MagicMock(major=3, minor=6))
    monkeypatch.setattr(prefect, "__version__", "0.9.2+c2394823")

    storage = Docker()

    assert not storage.registry_url
    assert storage.base_image == "python:3.6-slim"
    assert not storage.image_name
    assert not storage.image_tag
    assert storage.python_dependencies == ["wheel"]
    assert storage.env_vars == {
        "PREFECT__USER_CONFIG_PATH": "/root/.prefect/config.toml"
    }
    assert not storage.files
    assert storage.prefect_version
    assert storage.base_url == url
    assert not storage.local_image
    assert not storage.ignore_healthchecks
Exemplo n.º 7
0
def test_environment_execute():
    with tempfile.TemporaryDirectory() as directory:

        @prefect.task
        def add_to_dict():
            with open(path.join(directory, "output"), "w") as tmp:
                tmp.write("success")

        with open(path.join(directory, "flow_env.prefect"), "w+") as env:
            flow = prefect.Flow("test", tasks=[add_to_dict])
            flow_path = path.join(directory, "flow_env.prefect")
            with open(flow_path, "wb") as f:
                cloudpickle.dump(flow, f)

        environment = RemoteEnvironment()
        storage = Docker(registry_url="test")

        environment.execute(storage, flow_path)

        with open(path.join(directory, "output"), "r") as file:
            assert file.read() == "success"
Exemplo n.º 8
0
def test_docker_agent_deploy_flow_reg_allow_list_not_allowed(
        monkeypatch, cloud_api):
    api = MagicMock()
    api.ping.return_value = True
    api.create_container.return_value = {"Id": "container_id"}
    monkeypatch.setattr(
        "prefect.agent.docker.agent.DockerAgent._get_docker_client",
        MagicMock(return_value=api),
    )

    agent = DockerAgent(reg_allow_list=["test1"])

    with pytest.raises(ValueError) as error:
        agent.deploy_flow(flow_run=GraphQLResult({
            "flow":
            GraphQLResult({
                "id":
                "foo",
                "storage":
                Docker(registry_url="test2",
                       image_name="name",
                       image_tag="tag").serialize(),
                "environment":
                LocalEnvironment().serialize(),
                "core_version":
                "0.13.0",
            }),
            "id":
            "id",
            "name":
            "name",
        }))

    expected_error = ("Trying to pull image from a Docker registry 'test2'"
                      " which is not in the reg_allow_list")

    assert not api.pull.called
    assert not api.create_container.called
    assert not api.start.called
    assert str(error.value) == expected_error
Exemplo n.º 9
0
def test_docker_agent_deploy_flow(core_version, command, monkeypatch,
                                  cloud_api):
    api = MagicMock()
    api.ping.return_value = True
    api.create_container.return_value = {"Id": "container_id"}
    api.create_host_config.return_value = {"AutoRemove": True}
    monkeypatch.setattr(
        "prefect.agent.docker.agent.DockerAgent._get_docker_client",
        MagicMock(return_value=api),
    )

    agent = DockerAgent()
    agent.deploy_flow(flow_run=GraphQLResult({
        "flow":
        GraphQLResult({
            "id":
            "foo",
            "storage":
            Docker(registry_url="test", image_name="name",
                   image_tag="tag").serialize(),
            "environment":
            LocalEnvironment().serialize(),
            "core_version":
            core_version,
        }),
        "id":
        "id",
        "name":
        "name",
    }))

    assert api.pull.called
    assert api.create_container.called
    assert api.start.called

    assert api.create_host_config.call_args[1]["auto_remove"] is True
    assert api.create_container.call_args[1]["command"] == command
    assert api.create_container.call_args[1]["host_config"][
        "AutoRemove"] is True
    assert api.start.call_args[1]["container"] == "container_id"
Exemplo n.º 10
0
def test_k8s_agent_deploy_flow(core_version, command, monkeypatch, cloud_api):
    k8s_config = MagicMock()
    monkeypatch.setattr("kubernetes.config", k8s_config)

    batch_client = MagicMock()
    batch_client.create_namespaced_job.return_value = {}
    monkeypatch.setattr(
        "kubernetes.client.BatchV1Api", MagicMock(retrurn_value=batch_client)
    )

    agent = KubernetesAgent()
    agent.deploy_flow(
        flow_run=GraphQLResult(
            {
                "flow": GraphQLResult(
                    {
                        "storage": Docker(
                            registry_url="test", image_name="name", image_tag="tag"
                        ).serialize(),
                        "environment": LocalEnvironment().serialize(),
                        "id": "id",
                        "core_version": core_version,
                    }
                ),
                "id": "id",
            }
        )
    )

    assert agent.batch_client.create_namespaced_job.called
    assert (
        agent.batch_client.create_namespaced_job.call_args[1]["namespace"] == "default"
    )
    assert (
        agent.batch_client.create_namespaced_job.call_args[1]["body"]["apiVersion"]
        == "batch/v1"
    )
    assert agent.batch_client.create_namespaced_job.call_args[1]["body"]["spec"][
        "template"
    ]["spec"]["containers"][0]["args"] == [command]
Exemplo n.º 11
0
def test_docker_agent_deploy_flow_run_config(api, run_kind,
                                             has_docker_storage):
    if has_docker_storage:
        storage = Docker(registry_url="testing",
                         image_name="on-storage",
                         image_tag="tag")
        image = "testing/on-storage:tag"
    else:
        storage = Local()
        image = ("on-run-config" if run_kind == "docker" else
                 "prefecthq/prefect:all_extras-0.13.11")

    if run_kind == "docker":
        env = {"TESTING": "VALUE"}
        run = DockerRun(image=image, env=env)
    else:
        env = {}
        run = None if run_kind == "missing" else UniversalRun()

    agent = DockerAgent()
    agent.deploy_flow(flow_run=GraphQLResult({
        "flow":
        GraphQLResult({
            "id": "foo",
            "storage": storage.serialize(),
            "run_config": run.serialize() if run else None,
            "core_version": "0.13.11",
        }),
        "id":
        "id",
        "name":
        "name",
    }))

    assert api.create_container.called
    assert api.create_container.call_args[0][0] == image
    res_env = api.create_container.call_args[1]["environment"]
    for k, v in env.items():
        assert res_env[k] == v
Exemplo n.º 12
0
def test_execute(monkeypatch):
    with tempfile.TemporaryDirectory() as directory:

        with open(os.path.join(directory, "job.yaml"), "w+") as file:
            file.write("job")

        environment = KubernetesJobEnvironment(
            job_spec_file=os.path.join(directory, "job.yaml"))
        storage = Docker(registry_url="test1",
                         image_name="test2",
                         image_tag="test3")

        create_flow_run = MagicMock()
        monkeypatch.setattr(
            "prefect.environments.KubernetesJobEnvironment.create_flow_run_job",
            create_flow_run,
        )

        environment.execute(storage=storage, flow_location="")

        assert create_flow_run.call_args[1][
            "docker_name"] == "test1/test2:test3"
Exemplo n.º 13
0
def test_create_secret_isnt_called_if_exists(monkeypatch):
    environment = CloudEnvironment(private_registry=True)

    config = MagicMock()
    monkeypatch.setattr("kubernetes.config", config)

    secret = MagicMock()
    secret.metadata.name = "foo-docker"
    v1 = MagicMock()
    v1.list_namespaced_secret.return_value = MagicMock(items=[secret])
    monkeypatch.setattr("kubernetes.client",
                        MagicMock(CoreV1Api=MagicMock(return_value=v1)))

    create_secret = MagicMock()
    monkeypatch.setattr(
        "prefect.environments.CloudEnvironment._create_namespaced_secret",
        create_secret)
    with set_temporary_config({"cloud.auth_token": "test"}):
        with prefect.context(namespace="foo"):
            environment.setup(storage=Docker())

    assert not create_secret.called
Exemplo n.º 14
0
def test_build_image_passes(monkeypatch):
    flow = Flow("test")
    storage = Docker(
        registry_url="reg",
        base_image="python:3.6",
        image_name="test",
        image_tag="latest",
    )

    pull_image = MagicMock()
    monkeypatch.setattr("prefect.environments.storage.Docker.pull_image", pull_image)

    build = MagicMock()
    monkeypatch.setattr("docker.APIClient.build", build)

    images = MagicMock(return_value=["test"])
    monkeypatch.setattr("docker.APIClient.images", images)

    image_name, image_tag = storage._build_image(flow, push=False)

    assert image_name
    assert image_tag
Exemplo n.º 15
0
def test_dockerfile_env_vars(tmpdir):
    env_vars = OrderedDict([
        ("NUM", 1),
        ("STR_WITH_SPACES", "Hello world!"),
        ("STR_WITH_QUOTES", 'Hello "friend"'),
        ("STR_WITH_SINGLE_QUOTES", "'foo'"),
    ])
    storage = Docker(env_vars=env_vars, )
    storage.add_flow(Flow("foo"))
    dpath = storage.create_dockerfile_object(directory=str(tmpdir))

    with open(dpath, "r") as dockerfile:
        output = dockerfile.read()

    expected = textwrap.dedent("""
        ENV NUM=1 \\
            STR_WITH_SPACES='Hello world!' \\
            STR_WITH_QUOTES='Hello "friend"' \\
            STR_WITH_SINGLE_QUOTES="'foo'" \\
        """)

    assert expected in output
Exemplo n.º 16
0
def test_k8s_agent_replace_yaml_respects_multiple_image_secrets(
        monkeypatch, cloud_api):
    get_jobs = MagicMock(return_value=[])
    monkeypatch.setattr(
        "prefect.agent.kubernetes.agent.KubernetesAgent.manage_jobs",
        get_jobs,
    )

    monkeypatch.setenv("IMAGE_PULL_SECRETS", "some-secret,other-secret")
    monkeypatch.setenv("IMAGE_PULL_POLICY", "custom_policy")

    flow_run = GraphQLResult({
        "flow":
        GraphQLResult({
            "storage":
            Docker(registry_url="test", image_name="name",
                   image_tag="tag").serialize(),
            "environment":
            LocalEnvironment().serialize(),
            "id":
            "new_id",
            "core_version":
            "0.13.0",
        }),
        "id":
        "id",
    })

    with set_temporary_config({
            "cloud.agent.auth_token": "token",
            "logging.log_to_cloud": True
    }):
        agent = KubernetesAgent(
            env_vars=dict(AUTH_THING="foo", PKG_SETTING="bar"))
        job = agent.generate_job_spec_from_environment(flow_run,
                                                       image="test/name:tag")
        expected_secrets = [{"name": "some-secret"}, {"name": "other-secret"}]
        assert job["spec"]["template"]["spec"][
            "imagePullSecrets"] == expected_secrets
Exemplo n.º 17
0
def test_initialized_docker_storage():
    storage = Docker(
        registry_url="test1",
        base_image="test3",
        python_dependencies=["test"],
        image_name="test4",
        image_tag="test5",
        env_vars={"test": "1"},
        base_url="test_url",
        prefect_version="my-branch",
        local_image=True,
    )

    assert storage.registry_url == "test1"
    assert storage.base_image == "test3"
    assert storage.image_name == "test4"
    assert storage.image_tag == "test5"
    assert storage.python_dependencies == ["test"]
    assert storage.env_vars == {"test": "1"}
    assert storage.base_url == "test_url"
    assert storage.prefect_version == "my-branch"
    assert storage.local_image
Exemplo n.º 18
0
def test_k8s_agent_replace_yaml(monkeypatch, runner_token):
    k8s_config = MagicMock()
    monkeypatch.setattr("kubernetes.config", k8s_config)

    monkeypatch.setenv("IMAGE_PULL_SECRETS", "my-secret")

    flow_run = GraphQLResult({
        "flow":
        GraphQLResult({
            "storage":
            Docker(registry_url="test", image_name="name",
                   image_tag="tag").serialize(),
            "id":
            "id",
        }),
        "id":
        "id",
    })

    with set_temporary_config({"cloud.agent.auth_token": "token"}):
        agent = KubernetesAgent()
        job = agent.replace_job_spec_yaml(flow_run)

        assert job["metadata"]["labels"]["flow_run_id"] == "id"
        assert job["metadata"]["labels"]["flow_id"] == "id"
        assert job["spec"]["template"]["metadata"]["labels"][
            "flow_run_id"] == "id"
        assert (job["spec"]["template"]["spec"]["containers"][0]["image"] ==
                "test/name:tag")

        env = job["spec"]["template"]["spec"]["containers"][0]["env"]

        assert env[0]["value"] == "https://api.prefect.io"
        assert env[1]["value"] == "token"
        assert env[2]["value"] == "id"
        assert env[3]["value"] == "default"

        assert (job["spec"]["template"]["spec"]["imagePullSecrets"][0]["name"]
                == "my-secret")
Exemplo n.º 19
0
def test_execute(monkeypatch):
    environment = DaskKubernetesEnvironment()

    config = MagicMock()
    monkeypatch.setattr("kubernetes.config", config)

    batchv1 = MagicMock()
    monkeypatch.setattr("kubernetes.client",
                        MagicMock(BatchV1Api=MagicMock(return_value=batchv1)))

    environment = DaskKubernetesEnvironment()
    storage = Docker(registry_url="test1",
                     image_name="test2",
                     image_tag="test3")

    flow = base_flow
    flow.storage = storage
    with set_temporary_config({"cloud.auth_token": "test"}):
        environment.execute(flow=flow)

    assert (batchv1.create_namespaced_job.call_args[1]["body"]["apiVersion"] ==
            "batch/v1")
Exemplo n.º 20
0
def test_nomad_agent_deploy_flows(monkeypatch, runner_token):
    requests = MagicMock()
    monkeypatch.setattr("prefect.agent.nomad.agent.requests", requests)

    agent = NomadAgent()
    agent.deploy_flows(flow_runs=[
        GraphQLResult({
            "flow":
            GraphQLResult({
                "storage":
                Docker(registry_url="test", image_name="name",
                       image_tag="tag").serialize(),
                "id":
                "id",
            }),
            "id":
            "id",
        })
    ])

    assert requests.post.called
    assert requests.post.call_args[1]["json"]
Exemplo n.º 21
0
def test_docker_agent_deploy_flow_show_flow_logs(api, monkeypatch):
    process = MagicMock()
    monkeypatch.setattr("multiprocessing.Process", process)

    agent = DockerAgent(show_flow_logs=True)
    agent.deploy_flow(flow_run=GraphQLResult({
        "flow":
        GraphQLResult({
            "id":
            "foo",
            "storage":
            Docker(registry_url="test", image_name="name",
                   image_tag="tag").serialize(),
            "environment":
            LocalEnvironment().serialize(),
            "core_version":
            "0.13.0",
        }),
        "id":
        "id",
        "name":
        "name",
    }))

    process_kwargs = dict(
        target=_stream_container_logs,
        kwargs={
            "base_url": agent.base_url,
            "container_id": "container_id"
        },
    )
    process.assert_called_with(**process_kwargs)
    # Check all arguments to `multiprocessing.Process` are pickleable
    assert pickle.loads(pickle.dumps(process_kwargs)) == process_kwargs

    assert len(agent.processes) == 1
    assert api.create_container.called
    assert api.start.called
Exemplo n.º 22
0
def test_deploy_flow_register_task_definition_uses_user_env_vars(
        monkeypatch, runner_token):
    boto3_client = MagicMock()

    boto3_client.describe_task_definition.side_effect = ClientError({}, None)
    boto3_client.run_task.return_value = {"tasks": [{"taskArn": "test"}]}
    boto3_client.register_task_definition.return_value = {}

    monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client))

    agent = FargateAgent(env_vars=dict(AUTH_THING="foo", PKG_SETTING="bar"))
    agent.deploy_flow(flow_run=GraphQLResult({
        "flow":
        GraphQLResult({
            "storage":
            Docker(registry_url="test", image_name="name",
                   image_tag="tag").serialize(),
            "id":
            "id",
        }),
        "id":
        "id",
    }))

    assert boto3_client.describe_task_definition.called
    assert boto3_client.register_task_definition.called
    assert (boto3_client.register_task_definition.call_args[1]["family"] ==
            "prefect-task-id")

    container_defs = boto3_client.register_task_definition.call_args[1][
        "containerDefinitions"]

    user_vars = [
        dict(name="AUTH_THING", value="foo"),
        dict(name="PKG_SETTING", value="bar"),
    ]
    assert container_defs[0]["environment"][-1] in user_vars
    assert container_defs[0]["environment"][-2] in user_vars
Exemplo n.º 23
0
def test_docker_agent_deploy_flow_show_flow_logs(monkeypatch, cloud_api):

    process = MagicMock()
    monkeypatch.setattr("multiprocessing.Process", process)

    api = MagicMock()
    api.ping.return_value = True
    api.create_container.return_value = {"Id": "container_id"}
    monkeypatch.setattr(
        "prefect.agent.docker.agent.DockerAgent._get_docker_client",
        MagicMock(return_value=api),
    )

    agent = DockerAgent(show_flow_logs=True)
    agent.deploy_flow(flow_run=GraphQLResult({
        "flow":
        GraphQLResult({
            "id":
            "foo",
            "storage":
            Docker(registry_url="test", image_name="name",
                   image_tag="tag").serialize(),
            "environment":
            LocalEnvironment().serialize(),
            "core_version":
            "0.13.0",
        }),
        "id":
        "id",
        "name":
        "name",
    }))

    process.assert_called_with(target=agent.stream_container_logs,
                               kwargs={"container_id": "container_id"})
    assert len(agent.processes) == 1
    assert api.create_container.called
    assert api.start.called
def test_execute_run_task_agent_token(monkeypatch):
    boto3_client = MagicMock()
    boto3_client.run_task.return_value = {}
    monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client))

    with set_temporary_config({"cloud.agent.auth_token": "test"}):
        environment = FargateTaskEnvironment(
            cluster="test", family="test", taskDefinition="test"
        )

        environment.execute(
            Flow(
                "test",
                storage=Docker(
                    registry_url="test", image_name="image", image_tag="tag"
                ),
            ),
        )

        assert boto3_client.run_task.called
        assert boto3_client.run_task.call_args[1]["taskDefinition"] == "test"
        assert boto3_client.run_task.call_args[1]["overrides"] == {
            "containerOverrides": [
                {
                    "name": "flow-container",
                    "environment": [
                        {
                            "name": "PREFECT__CLOUD__AUTH_TOKEN",
                            "value": prefect.config.cloud.agent.get("auth_token"),
                        },
                        {"name": "PREFECT__CONTEXT__FLOW_RUN_ID", "value": "unknown"},
                        {"name": "PREFECT__CONTEXT__IMAGE", "value": "test/image:tag"},
                    ],
                }
            ]
        }
        assert boto3_client.run_task.call_args[1]["launchType"] == "FARGATE"
        assert boto3_client.run_task.call_args[1]["cluster"] == "test"
Exemplo n.º 25
0
def test_create_dockerfile_from_dockerfile_uses_tempdir_path():
    myfile = "FROM my-own-image:latest\n\nRUN echo 'hi'"
    with tempfile.TemporaryDirectory() as tempdir_outside:

        with open(os.path.join(tempdir_outside, "test"), "w+") as t:
            t.write("asdf")

        with tempfile.TemporaryDirectory() as directory:

            with open(os.path.join(directory, "myfile"), "w") as tmp:
                tmp.write(myfile)

            storage = Docker(
                dockerfile=os.path.join(directory, "myfile"),
                files={os.path.join(tempdir_outside, "test"): "./test2"},
            )
            storage.add_flow(Flow("foo"))
            dpath = storage.create_dockerfile_object(directory=directory)

            with open(dpath, "r") as dockerfile:
                output = dockerfile.read()

            assert ("COPY {} /opt/prefect/flows/foo.prefect".format(
                os.path.join(directory, "foo.flow").replace("\\", "/"))
                    in output), output
            assert ("COPY {} ./test2".format(
                os.path.join(directory, "test").replace("\\", "/"))
                    in output), output
            assert ("COPY {} /opt/prefect/healthcheck.py".format(
                os.path.join(directory, "healthcheck.py").replace("\\", "/"))
                    in output)

    assert output.startswith("\n" + myfile)

    # test proper indentation
    assert all(line == line.lstrip() for line in output.split("\n")
               if line not in ["\n", " "])
Exemplo n.º 26
0
def test_k8s_agent_deploy_flows(monkeypatch, runner_token):
    k8s_config = MagicMock()
    monkeypatch.setattr("kubernetes.config", k8s_config)

    batch_client = MagicMock()
    batch_client.create_namespaced_job.return_value = {}
    monkeypatch.setattr(
        "kubernetes.client.BatchV1Api", MagicMock(retrurn_value=batch_client)
    )

    agent = KubernetesAgent()
    agent.deploy_flows(
        flow_runs=[
            GraphQLResult(
                {
                    "flow": GraphQLResult(
                        {
                            "storage": Docker(
                                registry_url="test", image_name="name", image_tag="tag"
                            ).serialize(),
                            "id": "id",
                        }
                    ),
                    "id": "id",
                }
            )
        ]
    )

    assert agent.batch_client.create_namespaced_job.called
    assert (
        agent.batch_client.create_namespaced_job.call_args[1]["namespace"] == "default"
    )
    assert (
        agent.batch_client.create_namespaced_job.call_args[1]["body"]["apiVersion"]
        == "batch/v1"
    )
Exemplo n.º 27
0
def test_k8s_agent_replace_yaml_no_pull_secrets(monkeypatch, cloud_api):
    k8s_config = MagicMock()
    monkeypatch.setattr("kubernetes.config", k8s_config)

    flow_run = GraphQLResult(
        {
            "flow": GraphQLResult(
                {
                    "storage": Docker(
                        registry_url="test", image_name="name", image_tag="tag"
                    ).serialize(),
                    "environment": LocalEnvironment().serialize(),
                    "id": "id",
                    "core_version": "0.13.0",
                }
            ),
            "id": "id",
        }
    )

    agent = KubernetesAgent()
    job = agent.replace_job_spec_yaml(flow_run, image="test/name:tag")

    assert not job["spec"]["template"]["spec"].get("imagePullSecrets", None)
Exemplo n.º 28
0
def test_k8s_agent_includes_agent_labels_in_job(monkeypatch, cloud_api):
    k8s_config = MagicMock()
    monkeypatch.setattr("kubernetes.config", k8s_config)

    flow_run = GraphQLResult(
        {
            "flow": GraphQLResult(
                {
                    "storage": Docker(
                        registry_url="test", image_name="name", image_tag="tag"
                    ).serialize(),
                    "environment": LocalEnvironment().serialize(),
                    "id": "new_id",
                    "core_version": "0.13.0",
                }
            ),
            "id": "id",
        }
    )

    agent = KubernetesAgent(labels=["foo", "bar"])
    job = agent.replace_job_spec_yaml(flow_run, image="test/name:tag")
    env = job["spec"]["template"]["spec"]["containers"][0]["env"]
    assert env[5]["value"] == "['foo', 'bar']"
Exemplo n.º 29
0
def test_docker_agent_deploy_flow_no_registry_does_not_pull(api):
    agent = DockerAgent()
    agent.deploy_flow(
        flow_run=GraphQLResult(
            {
                "flow": GraphQLResult(
                    {
                        "id": "foo",
                        "storage": Docker(
                            registry_url="", image_name="name", image_tag="tag"
                        ).serialize(),
                        "environment": LocalEnvironment().serialize(),
                        "core_version": "0.13.0",
                    }
                ),
                "id": "id",
                "name": "name",
            }
        )
    )

    assert not api.pull.called
    assert api.create_container.called
    assert api.start.called
Exemplo n.º 30
0
def test_docker_agent_deploy_with_no_interface_check_linux(
    monkeypatch, cloud_api, linux_platform
):

    api = MagicMock()
    api.ping.return_value = True
    api.create_container.return_value = {"Id": "container_id"}
    monkeypatch.setattr(
        "prefect.agent.docker.agent.DockerAgent._get_docker_client",
        MagicMock(return_value=api),
    )

    get_ip = MagicMock()
    monkeypatch.setattr("prefect.agent.docker.agent.get_docker_ip", get_ip)

    agent = DockerAgent(docker_interface=False)
    agent.deploy_flow(
        flow_run=GraphQLResult(
            {
                "flow": GraphQLResult(
                    {
                        "id": "foo",
                        "storage": Docker(
                            registry_url="", image_name="name", image_tag="tag"
                        ).serialize(),
                        "environment": LocalEnvironment().serialize(),
                        "core_version": "0.13.0",
                    }
                ),
                "id": "id",
                "name": "name",
            }
        )
    )

    assert not get_ip.called