def test_k8s_agent_includes_agent_labels_in_job(monkeypatch, runner_token): k8s_config = MagicMock() monkeypatch.setattr("kubernetes.config", k8s_config) flow_run = GraphQLResult({ "flow": GraphQLResult({ "storage": Docker(registry_url="test", image_name="name", image_tag="tag").serialize(), "id": "id", }), "id": "id", }) agent = KubernetesAgent(labels=["foo", "bar"]) job = agent.replace_job_spec_yaml(flow_run) env = job["spec"]["template"]["spec"]["containers"][0]["env"] assert env[4]["value"] == "['foo', 'bar']"
def test_setup_doesnt_pass_if_private_registry(monkeypatch): environment = DaskKubernetesEnvironment(private_registry=True) assert environment.docker_secret == "DOCKER_REGISTRY_CREDENTIALS" config = MagicMock() monkeypatch.setattr("kubernetes.config", config) v1 = MagicMock() v1.list_namespaced_secret.return_value = MagicMock(items=[]) monkeypatch.setattr("kubernetes.client", MagicMock(CoreV1Api=MagicMock(return_value=v1))) create_secret = MagicMock() monkeypatch.setattr( "prefect.environments.DaskKubernetesEnvironment._create_namespaced_secret", create_secret, ) with set_temporary_config({"cloud.auth_token": "test"}): environment.setup(storage=Docker()) assert create_secret.called
def test_k8s_agent_replace_yaml_no_pull_secrets(monkeypatch, runner_token): k8s_config = MagicMock() monkeypatch.setattr("kubernetes.config", k8s_config) flow_run = GraphQLResult({ "flow": GraphQLResult({ "storage": Docker(registry_url="test", image_name="name", image_tag="tag").serialize(), "id": "id", }), "id": "id", }) agent = KubernetesAgent() job = agent.replace_job_spec_yaml(flow_run) assert not job["spec"]["template"]["spec"]["imagePullSecrets"][0]["name"]
def test_run_healthchecks_arg(ignore_healthchecks): with tempfile.TemporaryDirectory() as tempdir_outside: with open(os.path.join(tempdir_outside, "test"), "w+") as t: t.write("asdf") with tempfile.TemporaryDirectory() as tempdir: storage = Docker(ignore_healthchecks=ignore_healthchecks) f = Flow("test") storage.add_flow(f) dpath = storage.create_dockerfile_object(directory=tempdir) with open(dpath, "r") as dockerfile: output = dockerfile.read() if ignore_healthchecks: assert "RUN python /opt/prefect/healthcheck.py" not in output else: assert "RUN python /opt/prefect/healthcheck.py" in output
def test_empty_docker_storage_on_tagged_commit(monkeypatch, platform, url, no_docker_host_var): monkeypatch.setattr("prefect.environments.storage.docker.sys.platform", platform) monkeypatch.setattr(sys, "version_info", MagicMock(major=3, minor=6)) monkeypatch.setattr(prefect, "__version__", "0.9.2") storage = Docker() assert not storage.registry_url assert storage.base_image == "prefecthq/prefect:0.9.2-python3.6" assert not storage.image_name assert not storage.image_tag assert storage.python_dependencies == ["wheel"] assert storage.env_vars == { "PREFECT__USER_CONFIG_PATH": "/opt/prefect/config.toml" } assert not storage.files assert storage.prefect_version assert storage.base_url == url assert not storage.local_image
def test_empty_docker_storage(monkeypatch, platform, url): monkeypatch.setattr("prefect.environments.storage.docker.sys.platform", platform) monkeypatch.setattr(sys, "version_info", MagicMock(major=3, minor=6)) monkeypatch.setattr(prefect, "__version__", "0.9.2+c2394823") storage = Docker() assert not storage.registry_url assert storage.base_image == "python:3.6-slim" assert not storage.image_name assert not storage.image_tag assert storage.python_dependencies == ["wheel"] assert storage.env_vars == { "PREFECT__USER_CONFIG_PATH": "/root/.prefect/config.toml" } assert not storage.files assert storage.prefect_version assert storage.base_url == url assert not storage.local_image assert not storage.ignore_healthchecks
def test_environment_execute(): with tempfile.TemporaryDirectory() as directory: @prefect.task def add_to_dict(): with open(path.join(directory, "output"), "w") as tmp: tmp.write("success") with open(path.join(directory, "flow_env.prefect"), "w+") as env: flow = prefect.Flow("test", tasks=[add_to_dict]) flow_path = path.join(directory, "flow_env.prefect") with open(flow_path, "wb") as f: cloudpickle.dump(flow, f) environment = RemoteEnvironment() storage = Docker(registry_url="test") environment.execute(storage, flow_path) with open(path.join(directory, "output"), "r") as file: assert file.read() == "success"
def test_docker_agent_deploy_flow_reg_allow_list_not_allowed( monkeypatch, cloud_api): api = MagicMock() api.ping.return_value = True api.create_container.return_value = {"Id": "container_id"} monkeypatch.setattr( "prefect.agent.docker.agent.DockerAgent._get_docker_client", MagicMock(return_value=api), ) agent = DockerAgent(reg_allow_list=["test1"]) with pytest.raises(ValueError) as error: agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "storage": Docker(registry_url="test2", image_name="name", image_tag="tag").serialize(), "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", }), "id": "id", "name": "name", })) expected_error = ("Trying to pull image from a Docker registry 'test2'" " which is not in the reg_allow_list") assert not api.pull.called assert not api.create_container.called assert not api.start.called assert str(error.value) == expected_error
def test_docker_agent_deploy_flow(core_version, command, monkeypatch, cloud_api): api = MagicMock() api.ping.return_value = True api.create_container.return_value = {"Id": "container_id"} api.create_host_config.return_value = {"AutoRemove": True} monkeypatch.setattr( "prefect.agent.docker.agent.DockerAgent._get_docker_client", MagicMock(return_value=api), ) agent = DockerAgent() agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "storage": Docker(registry_url="test", image_name="name", image_tag="tag").serialize(), "environment": LocalEnvironment().serialize(), "core_version": core_version, }), "id": "id", "name": "name", })) assert api.pull.called assert api.create_container.called assert api.start.called assert api.create_host_config.call_args[1]["auto_remove"] is True assert api.create_container.call_args[1]["command"] == command assert api.create_container.call_args[1]["host_config"][ "AutoRemove"] is True assert api.start.call_args[1]["container"] == "container_id"
def test_k8s_agent_deploy_flow(core_version, command, monkeypatch, cloud_api): k8s_config = MagicMock() monkeypatch.setattr("kubernetes.config", k8s_config) batch_client = MagicMock() batch_client.create_namespaced_job.return_value = {} monkeypatch.setattr( "kubernetes.client.BatchV1Api", MagicMock(retrurn_value=batch_client) ) agent = KubernetesAgent() agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "storage": Docker( registry_url="test", image_name="name", image_tag="tag" ).serialize(), "environment": LocalEnvironment().serialize(), "id": "id", "core_version": core_version, } ), "id": "id", } ) ) assert agent.batch_client.create_namespaced_job.called assert ( agent.batch_client.create_namespaced_job.call_args[1]["namespace"] == "default" ) assert ( agent.batch_client.create_namespaced_job.call_args[1]["body"]["apiVersion"] == "batch/v1" ) assert agent.batch_client.create_namespaced_job.call_args[1]["body"]["spec"][ "template" ]["spec"]["containers"][0]["args"] == [command]
def test_docker_agent_deploy_flow_run_config(api, run_kind, has_docker_storage): if has_docker_storage: storage = Docker(registry_url="testing", image_name="on-storage", image_tag="tag") image = "testing/on-storage:tag" else: storage = Local() image = ("on-run-config" if run_kind == "docker" else "prefecthq/prefect:all_extras-0.13.11") if run_kind == "docker": env = {"TESTING": "VALUE"} run = DockerRun(image=image, env=env) else: env = {} run = None if run_kind == "missing" else UniversalRun() agent = DockerAgent() agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "storage": storage.serialize(), "run_config": run.serialize() if run else None, "core_version": "0.13.11", }), "id": "id", "name": "name", })) assert api.create_container.called assert api.create_container.call_args[0][0] == image res_env = api.create_container.call_args[1]["environment"] for k, v in env.items(): assert res_env[k] == v
def test_execute(monkeypatch): with tempfile.TemporaryDirectory() as directory: with open(os.path.join(directory, "job.yaml"), "w+") as file: file.write("job") environment = KubernetesJobEnvironment( job_spec_file=os.path.join(directory, "job.yaml")) storage = Docker(registry_url="test1", image_name="test2", image_tag="test3") create_flow_run = MagicMock() monkeypatch.setattr( "prefect.environments.KubernetesJobEnvironment.create_flow_run_job", create_flow_run, ) environment.execute(storage=storage, flow_location="") assert create_flow_run.call_args[1][ "docker_name"] == "test1/test2:test3"
def test_create_secret_isnt_called_if_exists(monkeypatch): environment = CloudEnvironment(private_registry=True) config = MagicMock() monkeypatch.setattr("kubernetes.config", config) secret = MagicMock() secret.metadata.name = "foo-docker" v1 = MagicMock() v1.list_namespaced_secret.return_value = MagicMock(items=[secret]) monkeypatch.setattr("kubernetes.client", MagicMock(CoreV1Api=MagicMock(return_value=v1))) create_secret = MagicMock() monkeypatch.setattr( "prefect.environments.CloudEnvironment._create_namespaced_secret", create_secret) with set_temporary_config({"cloud.auth_token": "test"}): with prefect.context(namespace="foo"): environment.setup(storage=Docker()) assert not create_secret.called
def test_build_image_passes(monkeypatch): flow = Flow("test") storage = Docker( registry_url="reg", base_image="python:3.6", image_name="test", image_tag="latest", ) pull_image = MagicMock() monkeypatch.setattr("prefect.environments.storage.Docker.pull_image", pull_image) build = MagicMock() monkeypatch.setattr("docker.APIClient.build", build) images = MagicMock(return_value=["test"]) monkeypatch.setattr("docker.APIClient.images", images) image_name, image_tag = storage._build_image(flow, push=False) assert image_name assert image_tag
def test_dockerfile_env_vars(tmpdir): env_vars = OrderedDict([ ("NUM", 1), ("STR_WITH_SPACES", "Hello world!"), ("STR_WITH_QUOTES", 'Hello "friend"'), ("STR_WITH_SINGLE_QUOTES", "'foo'"), ]) storage = Docker(env_vars=env_vars, ) storage.add_flow(Flow("foo")) dpath = storage.create_dockerfile_object(directory=str(tmpdir)) with open(dpath, "r") as dockerfile: output = dockerfile.read() expected = textwrap.dedent(""" ENV NUM=1 \\ STR_WITH_SPACES='Hello world!' \\ STR_WITH_QUOTES='Hello "friend"' \\ STR_WITH_SINGLE_QUOTES="'foo'" \\ """) assert expected in output
def test_k8s_agent_replace_yaml_respects_multiple_image_secrets( monkeypatch, cloud_api): get_jobs = MagicMock(return_value=[]) monkeypatch.setattr( "prefect.agent.kubernetes.agent.KubernetesAgent.manage_jobs", get_jobs, ) monkeypatch.setenv("IMAGE_PULL_SECRETS", "some-secret,other-secret") monkeypatch.setenv("IMAGE_PULL_POLICY", "custom_policy") flow_run = GraphQLResult({ "flow": GraphQLResult({ "storage": Docker(registry_url="test", image_name="name", image_tag="tag").serialize(), "environment": LocalEnvironment().serialize(), "id": "new_id", "core_version": "0.13.0", }), "id": "id", }) with set_temporary_config({ "cloud.agent.auth_token": "token", "logging.log_to_cloud": True }): agent = KubernetesAgent( env_vars=dict(AUTH_THING="foo", PKG_SETTING="bar")) job = agent.generate_job_spec_from_environment(flow_run, image="test/name:tag") expected_secrets = [{"name": "some-secret"}, {"name": "other-secret"}] assert job["spec"]["template"]["spec"][ "imagePullSecrets"] == expected_secrets
def test_initialized_docker_storage(): storage = Docker( registry_url="test1", base_image="test3", python_dependencies=["test"], image_name="test4", image_tag="test5", env_vars={"test": "1"}, base_url="test_url", prefect_version="my-branch", local_image=True, ) assert storage.registry_url == "test1" assert storage.base_image == "test3" assert storage.image_name == "test4" assert storage.image_tag == "test5" assert storage.python_dependencies == ["test"] assert storage.env_vars == {"test": "1"} assert storage.base_url == "test_url" assert storage.prefect_version == "my-branch" assert storage.local_image
def test_k8s_agent_replace_yaml(monkeypatch, runner_token): k8s_config = MagicMock() monkeypatch.setattr("kubernetes.config", k8s_config) monkeypatch.setenv("IMAGE_PULL_SECRETS", "my-secret") flow_run = GraphQLResult({ "flow": GraphQLResult({ "storage": Docker(registry_url="test", image_name="name", image_tag="tag").serialize(), "id": "id", }), "id": "id", }) with set_temporary_config({"cloud.agent.auth_token": "token"}): agent = KubernetesAgent() job = agent.replace_job_spec_yaml(flow_run) assert job["metadata"]["labels"]["flow_run_id"] == "id" assert job["metadata"]["labels"]["flow_id"] == "id" assert job["spec"]["template"]["metadata"]["labels"][ "flow_run_id"] == "id" assert (job["spec"]["template"]["spec"]["containers"][0]["image"] == "test/name:tag") env = job["spec"]["template"]["spec"]["containers"][0]["env"] assert env[0]["value"] == "https://api.prefect.io" assert env[1]["value"] == "token" assert env[2]["value"] == "id" assert env[3]["value"] == "default" assert (job["spec"]["template"]["spec"]["imagePullSecrets"][0]["name"] == "my-secret")
def test_execute(monkeypatch): environment = DaskKubernetesEnvironment() config = MagicMock() monkeypatch.setattr("kubernetes.config", config) batchv1 = MagicMock() monkeypatch.setattr("kubernetes.client", MagicMock(BatchV1Api=MagicMock(return_value=batchv1))) environment = DaskKubernetesEnvironment() storage = Docker(registry_url="test1", image_name="test2", image_tag="test3") flow = base_flow flow.storage = storage with set_temporary_config({"cloud.auth_token": "test"}): environment.execute(flow=flow) assert (batchv1.create_namespaced_job.call_args[1]["body"]["apiVersion"] == "batch/v1")
def test_nomad_agent_deploy_flows(monkeypatch, runner_token): requests = MagicMock() monkeypatch.setattr("prefect.agent.nomad.agent.requests", requests) agent = NomadAgent() agent.deploy_flows(flow_runs=[ GraphQLResult({ "flow": GraphQLResult({ "storage": Docker(registry_url="test", image_name="name", image_tag="tag").serialize(), "id": "id", }), "id": "id", }) ]) assert requests.post.called assert requests.post.call_args[1]["json"]
def test_docker_agent_deploy_flow_show_flow_logs(api, monkeypatch): process = MagicMock() monkeypatch.setattr("multiprocessing.Process", process) agent = DockerAgent(show_flow_logs=True) agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "storage": Docker(registry_url="test", image_name="name", image_tag="tag").serialize(), "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", }), "id": "id", "name": "name", })) process_kwargs = dict( target=_stream_container_logs, kwargs={ "base_url": agent.base_url, "container_id": "container_id" }, ) process.assert_called_with(**process_kwargs) # Check all arguments to `multiprocessing.Process` are pickleable assert pickle.loads(pickle.dumps(process_kwargs)) == process_kwargs assert len(agent.processes) == 1 assert api.create_container.called assert api.start.called
def test_deploy_flow_register_task_definition_uses_user_env_vars( monkeypatch, runner_token): boto3_client = MagicMock() boto3_client.describe_task_definition.side_effect = ClientError({}, None) boto3_client.run_task.return_value = {"tasks": [{"taskArn": "test"}]} boto3_client.register_task_definition.return_value = {} monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client)) agent = FargateAgent(env_vars=dict(AUTH_THING="foo", PKG_SETTING="bar")) agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "storage": Docker(registry_url="test", image_name="name", image_tag="tag").serialize(), "id": "id", }), "id": "id", })) assert boto3_client.describe_task_definition.called assert boto3_client.register_task_definition.called assert (boto3_client.register_task_definition.call_args[1]["family"] == "prefect-task-id") container_defs = boto3_client.register_task_definition.call_args[1][ "containerDefinitions"] user_vars = [ dict(name="AUTH_THING", value="foo"), dict(name="PKG_SETTING", value="bar"), ] assert container_defs[0]["environment"][-1] in user_vars assert container_defs[0]["environment"][-2] in user_vars
def test_docker_agent_deploy_flow_show_flow_logs(monkeypatch, cloud_api): process = MagicMock() monkeypatch.setattr("multiprocessing.Process", process) api = MagicMock() api.ping.return_value = True api.create_container.return_value = {"Id": "container_id"} monkeypatch.setattr( "prefect.agent.docker.agent.DockerAgent._get_docker_client", MagicMock(return_value=api), ) agent = DockerAgent(show_flow_logs=True) agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "storage": Docker(registry_url="test", image_name="name", image_tag="tag").serialize(), "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", }), "id": "id", "name": "name", })) process.assert_called_with(target=agent.stream_container_logs, kwargs={"container_id": "container_id"}) assert len(agent.processes) == 1 assert api.create_container.called assert api.start.called
def test_execute_run_task_agent_token(monkeypatch): boto3_client = MagicMock() boto3_client.run_task.return_value = {} monkeypatch.setattr("boto3.client", MagicMock(return_value=boto3_client)) with set_temporary_config({"cloud.agent.auth_token": "test"}): environment = FargateTaskEnvironment( cluster="test", family="test", taskDefinition="test" ) environment.execute( Flow( "test", storage=Docker( registry_url="test", image_name="image", image_tag="tag" ), ), ) assert boto3_client.run_task.called assert boto3_client.run_task.call_args[1]["taskDefinition"] == "test" assert boto3_client.run_task.call_args[1]["overrides"] == { "containerOverrides": [ { "name": "flow-container", "environment": [ { "name": "PREFECT__CLOUD__AUTH_TOKEN", "value": prefect.config.cloud.agent.get("auth_token"), }, {"name": "PREFECT__CONTEXT__FLOW_RUN_ID", "value": "unknown"}, {"name": "PREFECT__CONTEXT__IMAGE", "value": "test/image:tag"}, ], } ] } assert boto3_client.run_task.call_args[1]["launchType"] == "FARGATE" assert boto3_client.run_task.call_args[1]["cluster"] == "test"
def test_create_dockerfile_from_dockerfile_uses_tempdir_path(): myfile = "FROM my-own-image:latest\n\nRUN echo 'hi'" with tempfile.TemporaryDirectory() as tempdir_outside: with open(os.path.join(tempdir_outside, "test"), "w+") as t: t.write("asdf") with tempfile.TemporaryDirectory() as directory: with open(os.path.join(directory, "myfile"), "w") as tmp: tmp.write(myfile) storage = Docker( dockerfile=os.path.join(directory, "myfile"), files={os.path.join(tempdir_outside, "test"): "./test2"}, ) storage.add_flow(Flow("foo")) dpath = storage.create_dockerfile_object(directory=directory) with open(dpath, "r") as dockerfile: output = dockerfile.read() assert ("COPY {} /opt/prefect/flows/foo.prefect".format( os.path.join(directory, "foo.flow").replace("\\", "/")) in output), output assert ("COPY {} ./test2".format( os.path.join(directory, "test").replace("\\", "/")) in output), output assert ("COPY {} /opt/prefect/healthcheck.py".format( os.path.join(directory, "healthcheck.py").replace("\\", "/")) in output) assert output.startswith("\n" + myfile) # test proper indentation assert all(line == line.lstrip() for line in output.split("\n") if line not in ["\n", " "])
def test_k8s_agent_deploy_flows(monkeypatch, runner_token): k8s_config = MagicMock() monkeypatch.setattr("kubernetes.config", k8s_config) batch_client = MagicMock() batch_client.create_namespaced_job.return_value = {} monkeypatch.setattr( "kubernetes.client.BatchV1Api", MagicMock(retrurn_value=batch_client) ) agent = KubernetesAgent() agent.deploy_flows( flow_runs=[ GraphQLResult( { "flow": GraphQLResult( { "storage": Docker( registry_url="test", image_name="name", image_tag="tag" ).serialize(), "id": "id", } ), "id": "id", } ) ] ) assert agent.batch_client.create_namespaced_job.called assert ( agent.batch_client.create_namespaced_job.call_args[1]["namespace"] == "default" ) assert ( agent.batch_client.create_namespaced_job.call_args[1]["body"]["apiVersion"] == "batch/v1" )
def test_k8s_agent_replace_yaml_no_pull_secrets(monkeypatch, cloud_api): k8s_config = MagicMock() monkeypatch.setattr("kubernetes.config", k8s_config) flow_run = GraphQLResult( { "flow": GraphQLResult( { "storage": Docker( registry_url="test", image_name="name", image_tag="tag" ).serialize(), "environment": LocalEnvironment().serialize(), "id": "id", "core_version": "0.13.0", } ), "id": "id", } ) agent = KubernetesAgent() job = agent.replace_job_spec_yaml(flow_run, image="test/name:tag") assert not job["spec"]["template"]["spec"].get("imagePullSecrets", None)
def test_k8s_agent_includes_agent_labels_in_job(monkeypatch, cloud_api): k8s_config = MagicMock() monkeypatch.setattr("kubernetes.config", k8s_config) flow_run = GraphQLResult( { "flow": GraphQLResult( { "storage": Docker( registry_url="test", image_name="name", image_tag="tag" ).serialize(), "environment": LocalEnvironment().serialize(), "id": "new_id", "core_version": "0.13.0", } ), "id": "id", } ) agent = KubernetesAgent(labels=["foo", "bar"]) job = agent.replace_job_spec_yaml(flow_run, image="test/name:tag") env = job["spec"]["template"]["spec"]["containers"][0]["env"] assert env[5]["value"] == "['foo', 'bar']"
def test_docker_agent_deploy_flow_no_registry_does_not_pull(api): agent = DockerAgent() agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "id": "foo", "storage": Docker( registry_url="", image_name="name", image_tag="tag" ).serialize(), "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", } ), "id": "id", "name": "name", } ) ) assert not api.pull.called assert api.create_container.called assert api.start.called
def test_docker_agent_deploy_with_no_interface_check_linux( monkeypatch, cloud_api, linux_platform ): api = MagicMock() api.ping.return_value = True api.create_container.return_value = {"Id": "container_id"} monkeypatch.setattr( "prefect.agent.docker.agent.DockerAgent._get_docker_client", MagicMock(return_value=api), ) get_ip = MagicMock() monkeypatch.setattr("prefect.agent.docker.agent.get_docker_ip", get_ip) agent = DockerAgent(docker_interface=False) agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "id": "foo", "storage": Docker( registry_url="", image_name="name", image_tag="tag" ).serialize(), "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", } ), "id": "id", "name": "name", } ) ) assert not get_ip.called