def test_docker_agent_deploy_flow_no_pull_using_environment_metadata(api): agent = DockerAgent(no_pull=True) agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "name": "flow-name", "storage": Local().serialize(), "environment": LocalEnvironment(metadata={ "image": "name:tag" }).serialize(), "core_version": "0.13.0", }), "id": "id", "name": "name", })) assert not api.pull.called assert api.create_container.called assert api.start.called
def test_docker_agent_deploy_with_no_interface_check_linux( api, monkeypatch, linux_platform ): get_ip = MagicMock() monkeypatch.setattr("prefect.agent.docker.agent.get_docker_ip", get_ip) agent = DockerAgent(docker_interface=False) agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "id": "foo", "storage": Docker( registry_url="", image_name="name", image_tag="tag" ).serialize(), "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", } ), "id": "id", "name": "name", } ) ) assert not get_ip.called
def test_docker_agent_deploy_flow_storage_raises(monkeypatch, cloud_api): monkeypatch.setattr("prefect.agent.agent.Client", MagicMock()) api = MagicMock() api.ping.return_value = True api.create_container.return_value = {"Id": "container_id"} monkeypatch.setattr( "prefect.agent.docker.agent.DockerAgent._get_docker_client", MagicMock(return_value=api), ) agent = DockerAgent() with pytest.raises(ValueError): agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "storage": Local().serialize(), "id": "foo", "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", } ), "id": "id", "name": "name", "version": "version", } ) ) assert not api.pull.called
def test_docker_agent_deploy_with_no_interface_check_linux( monkeypatch, cloud_api, linux_platform): api = MagicMock() api.ping.return_value = True api.create_container.return_value = {"Id": "container_id"} monkeypatch.setattr( "prefect.agent.docker.agent.DockerAgent._get_docker_client", MagicMock(return_value=api), ) get_ip = MagicMock() monkeypatch.setattr("prefect.agent.docker.agent.get_docker_ip", get_ip) agent = DockerAgent(docker_interface=False) agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "storage": Docker(registry_url="", image_name="name", image_tag="tag").serialize(), "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", }), "id": "id", "name": "name", })) assert not get_ip.called
def test_docker_agent_deploy_flow_does_not_include_host_gateway_for_old_engine_versions( api, docker_engine_version): api.version.return_value = {"Version": docker_engine_version} run = UniversalRun() storage = Local() agent = DockerAgent() with pytest.warns( UserWarning, match= ("`host.docker.internal` could not be automatically resolved.*" f"feature is not supported on Docker Engine v{docker_engine_version}" ), ): agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "name": "flow-name", "storage": storage.serialize(), "core_version": "0.13.11", }), "run_config": run.serialize() if run else None, "id": "id", "name": "name", })) assert "extra_hosts" not in api.create_host_config.call_args[1]
def test_docker_agent_network(api): api.create_networking_config.return_value = {"test-network": "config"} agent = DockerAgent(network="test-network") agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "id": "foo", "storage": Docker( registry_url="test", image_name="name", image_tag="tag" ).serialize(), "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", } ), "id": "id", "name": "name", } ) ) assert agent.network == "test-network" args, kwargs = api.create_container.call_args assert kwargs["networking_config"] == {"test-network": "config"}
def test_docker_agent_deploy_flow_reg_allow_list_allowed(api): agent = DockerAgent(reg_allow_list=["test1"]) agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "id": "foo", "storage": Docker( registry_url="test1", image_name="name", image_tag="tag" ).serialize(), "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", } ), "id": "id", "name": "name", } ) ) assert api.pull.called assert api.create_container.called assert api.start.called
def test_docker_agent_networks_as_modes(api, network): api.create_networking_config.return_value = {network: "config1"} api.create_endpoint_config.return_value = "endpoint-config" agent = DockerAgent(networks=[network]) agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "name": "flow-name", "storage": Docker(registry_url="test", image_name="name", image_tag="tag").serialize(), "core_version": "0.13.0", }), "id": "id", "name": "name", })) assert network in agent.networks api.create_networking_config.assert_called_once_with( {network: "endpoint-config"}) _, container_create_kwargs = api.create_container.call_args assert container_create_kwargs["networking_config"] == {network: "config1"} _, host_config_kwargs = api.create_host_config.call_args assert host_config_kwargs["network_mode"] == network
def test_docker_agent_deploy_flow_unsupported_run_config(api): agent = DockerAgent() with pytest.raises( TypeError, match= "`run_config` of type `LocalRun`, only `DockerRun` is supported", ): agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "storage": Local().serialize(), "id": "foo", "name": "flow-name", "core_version": "0.13.0", }), "run_config": LocalRun().serialize(), "id": "id", "name": "name", "version": "version", })) assert not api.pull.called
def test_docker_agent_deploy_flow_run_config(api, image_on_run_config): if image_on_run_config: storage = Local() image = "on-run-config" run = DockerRun(image=image, env={"TESTING": "VALUE"}) else: storage = Docker( registry_url="testing", image_name="on-storage", image_tag="tag" ) image = "testing/on-storage:tag" run = DockerRun(env={"TESTING": "VALUE"}) agent = DockerAgent() agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "id": "foo", "storage": storage.serialize(), "run_config": run.serialize(), "core_version": "0.13.11", } ), "id": "id", "name": "name", } ) ) assert api.create_container.called assert api.create_container.call_args[0][0] == image assert api.create_container.call_args[1]["environment"]["TESTING"] == "VALUE"
def test_docker_agent_deploy_flow_reg_allow_list_not_allowed(api): agent = DockerAgent(reg_allow_list=["test1"]) with pytest.raises(ValueError) as error: agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "id": "foo", "storage": Docker( registry_url="test2", image_name="name", image_tag="tag" ).serialize(), "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", } ), "id": "id", "name": "name", } ) ) expected_error = ( "Trying to pull image from a Docker registry 'test2'" " which is not in the reg_allow_list" ) assert not api.pull.called assert not api.create_container.called assert not api.start.called assert str(error.value) == expected_error
def test_docker_agent_deploy_flow(core_version, command, api): agent = DockerAgent() agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "id": "foo", "storage": Docker( registry_url="test", image_name="name", image_tag="tag" ).serialize(), "environment": LocalEnvironment().serialize(), "core_version": core_version, } ), "id": "id", "name": "name", } ) ) assert api.pull.called assert api.create_container.called assert api.start.called assert api.create_host_config.call_args[1]["auto_remove"] is True assert api.create_container.call_args[1]["command"] == command assert api.create_container.call_args[1]["host_config"]["AutoRemove"] is True assert api.start.call_args[1]["container"] == "container_id"
def test_docker_agent_deploy_flow_uses_environment_metadata(api): agent = DockerAgent() agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "id": "foo", "storage": Local().serialize(), "environment": LocalEnvironment( metadata={"image": "repo/name:tag"} ).serialize(), "core_version": "0.13.0", } ), "id": "id", "name": "name", } ) ) assert api.pull.called assert api.create_container.called assert api.start.called assert api.create_host_config.call_args[1]["auto_remove"] is True assert api.create_container.call_args[1]["command"] == "prefect execute flow-run" assert api.create_container.call_args[1]["host_config"]["AutoRemove"] is True assert api.start.call_args[1]["container"] == "container_id"
def test_docker_agent_deploy_flow_no_registry_does_not_pull(monkeypatch, cloud_api): api = MagicMock() api.ping.return_value = True api.create_container.return_value = {"Id": "container_id"} monkeypatch.setattr( "prefect.agent.docker.agent.DockerAgent._get_docker_client", MagicMock(return_value=api), ) agent = DockerAgent() agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "id": "foo", "storage": Docker( registry_url="", image_name="name", image_tag="tag" ).serialize(), "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", } ), "id": "id", "name": "name", } ) ) assert not api.pull.called assert api.create_container.called assert api.start.called
def test_docker_agent_network(monkeypatch, cloud_api): api = MagicMock() api.ping.return_value = True api.create_container.return_value = {"Id": "container_id"} api.create_networking_config.return_value = {"test-network": "config"} monkeypatch.setattr( "prefect.agent.docker.agent.DockerAgent._get_docker_client", MagicMock(return_value=api), ) agent = DockerAgent(network="test-network") agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "id": "foo", "storage": Docker( registry_url="test", image_name="name", image_tag="tag" ).serialize(), "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", } ), "id": "id", "name": "name", } ) ) assert agent.network == "test-network" args, kwargs = api.create_container.call_args assert kwargs["networking_config"] == {"test-network": "config"}
def test_docker_agent_networks(api): api.create_networking_config.return_value = { "test-network-1": "config1", "test-network-2": "config2", } agent = DockerAgent(networks=["test-network-1", "test-network-2"]) agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "name": "flow-name", "storage": Docker(registry_url="test", image_name="name", image_tag="tag").serialize(), "core_version": "0.13.0", }), "id": "id", "name": "name", })) assert "test-network-1" in agent.networks assert "test-network-2" in agent.networks args, kwargs = api.create_container.call_args assert kwargs["networking_config"] == { "test-network-1": "config1", "test-network-2": "config2", }
def test_docker_agent_networks_no_networks(api): api.create_networking_config.return_value = {"test-network-1": "config1"} api.create_endpoint_config.return_value = "endpoint-config" agent = DockerAgent(networks=[]) agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "name": "flow-name", "storage": Docker(registry_url="test", image_name="name", image_tag="tag").serialize(), "core_version": "0.13.0", }), "id": "id", "name": "name", })) assert agent.networks == [] api.create_networking_config.assert_not_called() _, kwargs = api.create_container.call_args assert kwargs["networking_config"] is None api.connect_container_to_network.assert_not_called()
def test_docker_agent_deploy_flow_show_flow_logs(api, monkeypatch): process = MagicMock() monkeypatch.setattr("multiprocessing.Process", process) agent = DockerAgent(show_flow_logs=True) agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "id": "foo", "storage": Docker( registry_url="test", image_name="name", image_tag="tag" ).serialize(), "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", } ), "id": "id", "name": "name", } ) ) process_kwargs = dict( target=_stream_container_logs, kwargs={"base_url": agent.base_url, "container_id": "container_id"}, ) process.assert_called_with(**process_kwargs) # Check all arguments to `multiprocessing.Process` are pickleable assert pickle.loads(pickle.dumps(process_kwargs)) == process_kwargs assert len(agent.processes) == 1 assert api.create_container.called assert api.start.called
def test_docker_agent_deploy_flow_storage_raises(monkeypatch, api): monkeypatch.setattr("prefect.agent.agent.Client", MagicMock()) agent = DockerAgent() with pytest.raises(ValueError): agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "storage": Local().serialize(), "id": "foo", "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", } ), "id": "id", "name": "name", "version": "version", } ) ) assert not api.pull.called
def test_docker_agent_deploy_flow_sets_container_name_with_slugify( api, run_name, container_name): """ Asserts that the container name is set to the flow run name and that collisions with existing containers with the same name is handled by adding an index """ agent = DockerAgent() agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "name": "flow-name", "storage": Local().serialize(), "environment": LocalEnvironment(metadata={ "image": "repo/name:tag" }).serialize(), "core_version": "0.13.0", }), "id": "id", "name": run_name, })) assert api.create_container.call_args[1]["name"] == container_name
def test_docker_agent_deploy_flow_run_config(api, run_kind, has_docker_storage): if has_docker_storage: storage = Docker(registry_url="testing", image_name="on-storage", image_tag="tag") image = "testing/on-storage:tag" else: storage = Local() image = "on-run-config" if run_kind == "docker" else "prefecthq/prefect:0.13.11" if run_kind == "docker": env = {"TESTING": "VALUE"} host_config = {"auto_remove": False, "shm_size": "128m"} exp_host_config = { "auto_remove": False, "extra_hosts": { "host.docker.internal": "host-gateway" }, "shm_size": "128m", } run = DockerRun(image=image, env=env, host_config=host_config) else: env = {} host_config = {} exp_host_config = { "auto_remove": True, "extra_hosts": { "host.docker.internal": "host-gateway" }, } run = None if run_kind == "missing" else UniversalRun() agent = DockerAgent() agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "name": "flow-name", "storage": storage.serialize(), "core_version": "0.13.11", }), "run_config": run.serialize() if run else None, "id": "id", "name": "name", })) assert api.create_container.called assert api.create_container.call_args[0][0] == image res_env = api.create_container.call_args[1]["environment"] for k, v in env.items(): assert res_env[k] == v res_host_config = api.create_host_config.call_args[1] for k, v in exp_host_config.items(): assert res_host_config[k] == v
def test_docker_agent_deploy_flow_sets_container_name_with_index( api, collision_count): """ Asserts that the container name is set to the flow run name and that collisions with existing containers with the same name is handled by adding an index """ if collision_count: # Add the basic name first existing_names = ["flow-run-name"] for i in range(1, collision_count): existing_names.append(f"flow-run-name-{i}") else: existing_names = [] def fail_if_name_exists(*args, **kwargs): if kwargs.get("name") in existing_names: raise docker.errors.APIError( "Conflict. The container name 'foobar' is already in use") return {} api.create_container = MagicMock(side_effect=fail_if_name_exists) agent = DockerAgent() agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "name": "flow-name", "storage": Local().serialize(), "environment": LocalEnvironment(metadata={ "image": "repo/name:tag" }).serialize(), "core_version": "0.13.0", }), "id": "id", "name": "flow-run-name", })) expected_name = ("flow-run-name" if not collision_count else f"flow-run-name-{collision_count}") assert api.create_container.call_args[1]["name"] == expected_name
def test_docker_agent_deploy_flow_uses_environment_metadata( monkeypatch, cloud_api): api = MagicMock() api.ping.return_value = True api.create_container.return_value = {"Id": "container_id"} api.create_host_config.return_value = {"AutoRemove": True} monkeypatch.setattr( "prefect.agent.docker.agent.DockerAgent._get_docker_client", MagicMock(return_value=api), ) agent = DockerAgent() agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "storage": Local().serialize(), "environment": LocalEnvironment(metadata={ "image": "repo/name:tag" }).serialize(), "core_version": "0.13.0", }), "id": "id", "name": "name", })) assert api.pull.called assert api.create_container.called assert api.start.called assert api.create_host_config.call_args[1]["auto_remove"] is True assert api.create_container.call_args[1][ "command"] == "prefect execute flow-run" assert api.create_container.call_args[1]["host_config"][ "AutoRemove"] is True assert api.start.call_args[1]["container"] == "container_id"
def test_docker_agent_deploy_flow_run_config(api, run_kind, has_docker_storage): if has_docker_storage: storage = Docker( registry_url="testing", image_name="on-storage", image_tag="tag" ) image = "testing/on-storage:tag" else: storage = Local() image = "on-run-config" if run_kind == "docker" else "prefecthq/prefect:0.13.11" if run_kind == "docker": env = {"TESTING": "VALUE"} run = DockerRun(image=image, env=env) else: env = {} run = None if run_kind == "missing" else UniversalRun() agent = DockerAgent() agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "id": "foo", "name": "flow-name", "storage": storage.serialize(), "core_version": "0.13.11", } ), "run_config": run.serialize() if run else None, "id": "id", "name": "name", } ) ) assert api.create_container.called assert api.create_container.call_args[0][0] == image res_env = api.create_container.call_args[1]["environment"] for k, v in env.items(): assert res_env[k] == v
def test_docker_agent_deploy_flow(core_version, command, monkeypatch, cloud_api): api = MagicMock() api.ping.return_value = True api.create_container.return_value = {"Id": "container_id"} api.create_host_config.return_value = {"AutoRemove": True} monkeypatch.setattr( "prefect.agent.docker.agent.DockerAgent._get_docker_client", MagicMock(return_value=api), ) agent = DockerAgent() agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "storage": Docker(registry_url="test", image_name="name", image_tag="tag").serialize(), "environment": LocalEnvironment().serialize(), "core_version": core_version, }), "id": "id", "name": "name", })) assert api.pull.called assert api.create_container.called assert api.start.called assert api.create_host_config.call_args[1]["auto_remove"] is True assert api.create_container.call_args[1]["command"] == command assert api.create_container.call_args[1]["host_config"][ "AutoRemove"] is True assert api.start.call_args[1]["container"] == "container_id"
def test_docker_agent_deploy_flow_reg_allow_list_not_allowed(monkeypatch, cloud_api): api = MagicMock() api.ping.return_value = True api.create_container.return_value = {"Id": "container_id"} monkeypatch.setattr( "prefect.agent.docker.agent.DockerAgent._get_docker_client", MagicMock(return_value=api), ) agent = DockerAgent(reg_allow_list=["test1"]) with pytest.raises(ValueError) as error: agent.deploy_flow( flow_run=GraphQLResult( { "flow": GraphQLResult( { "id": "foo", "storage": Docker( registry_url="test2", image_name="name", image_tag="tag" ).serialize(), "environment": LocalEnvironment().serialize(), "core_version": "0.13.0", } ), "id": "id", "name": "name", } ) ) expected_error = ( "Trying to pull image from a Docker registry 'test2'" " which is not in the reg_allow_list" ) assert not api.pull.called assert not api.create_container.called assert not api.start.called assert str(error.value) == expected_error
def test_docker_agent_networks(api): api.create_networking_config.return_value = {"test-network-1": "config1"} api.create_endpoint_config.return_value = "endpoint-config" agent = DockerAgent(networks=["test-network-1", "test-network-2"]) agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "name": "flow-name", "storage": Docker(registry_url="test", image_name="name", image_tag="tag").serialize(), "core_version": "0.13.0", }), "id": "id", "name": "name", })) assert "test-network-1" in agent.networks assert "test-network-2" in agent.networks api.create_networking_config.assert_called_once_with( {"test-network-1": "endpoint-config"}) _, kwargs = api.create_container.call_args assert kwargs["networking_config"] == {"test-network-1": "config1"} api.connect_container_to_network.assert_called_once_with( container=ANY, net_id="test-network-2") _, host_config_kwargs = api.create_host_config.call_args assert "network_mode" not in host_config_kwargs
def test_docker_agent_deploy_flow_no_registry_does_not_pull(api): agent = DockerAgent() agent.deploy_flow(flow_run=GraphQLResult({ "flow": GraphQLResult({ "id": "foo", "name": "flow-name", "storage": Docker(registry_url="", image_name="name", image_tag="tag").serialize(), "core_version": "0.13.0", }), "id": "id", "name": "name", })) assert not api.pull.called assert api.create_container.called assert api.start.called