def test_docker_agent_deploy_flow_no_pull_using_environment_metadata(api):
agent = DockerAgent(no_pull=True)
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"id":
"foo",
"name":
"flow-name",
"storage":
Local().serialize(),
"environment":
LocalEnvironment(metadata={
"image": "name:tag"
}).serialize(),
"core_version":
"0.13.0",
}),
"id":
"id",
"name":
"name",
}))
assert not api.pull.called
assert api.create_container.called
assert api.start.called
def test_docker_agent_deploy_with_no_interface_check_linux(
api, monkeypatch, linux_platform
):
get_ip = MagicMock()
monkeypatch.setattr("prefect.agent.docker.agent.get_docker_ip", get_ip)
agent = DockerAgent(docker_interface=False)
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"id": "foo",
"storage": Docker(
registry_url="", image_name="name", image_tag="tag"
).serialize(),
"environment": LocalEnvironment().serialize(),
"core_version": "0.13.0",
}
),
"id": "id",
"name": "name",
}
)
)
assert not get_ip.called
def test_docker_agent_deploy_flow_storage_raises(monkeypatch, cloud_api):
monkeypatch.setattr("prefect.agent.agent.Client", MagicMock())
api = MagicMock()
api.ping.return_value = True
api.create_container.return_value = {"Id": "container_id"}
monkeypatch.setattr(
"prefect.agent.docker.agent.DockerAgent._get_docker_client",
MagicMock(return_value=api),
)
agent = DockerAgent()
with pytest.raises(ValueError):
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"storage": Local().serialize(),
"id": "foo",
"environment": LocalEnvironment().serialize(),
"core_version": "0.13.0",
}
),
"id": "id",
"name": "name",
"version": "version",
}
)
)
assert not api.pull.called
def test_docker_agent_deploy_with_no_interface_check_linux(
monkeypatch, cloud_api, linux_platform):
api = MagicMock()
api.ping.return_value = True
api.create_container.return_value = {"Id": "container_id"}
monkeypatch.setattr(
"prefect.agent.docker.agent.DockerAgent._get_docker_client",
MagicMock(return_value=api),
)
get_ip = MagicMock()
monkeypatch.setattr("prefect.agent.docker.agent.get_docker_ip", get_ip)
agent = DockerAgent(docker_interface=False)
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"id":
"foo",
"storage":
Docker(registry_url="", image_name="name",
image_tag="tag").serialize(),
"environment":
LocalEnvironment().serialize(),
"core_version":
"0.13.0",
}),
"id":
"id",
"name":
"name",
}))
assert not get_ip.called
def test_docker_agent_deploy_flow_does_not_include_host_gateway_for_old_engine_versions(
api, docker_engine_version):
api.version.return_value = {"Version": docker_engine_version}
run = UniversalRun()
storage = Local()
agent = DockerAgent()
with pytest.warns(
UserWarning,
match=
("`host.docker.internal` could not be automatically resolved.*"
f"feature is not supported on Docker Engine v{docker_engine_version}"
),
):
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"id": "foo",
"name": "flow-name",
"storage": storage.serialize(),
"core_version": "0.13.11",
}),
"run_config":
run.serialize() if run else None,
"id":
"id",
"name":
"name",
}))
assert "extra_hosts" not in api.create_host_config.call_args[1]
def test_docker_agent_network(api):
api.create_networking_config.return_value = {"test-network": "config"}
agent = DockerAgent(network="test-network")
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"id": "foo",
"storage": Docker(
registry_url="test", image_name="name", image_tag="tag"
).serialize(),
"environment": LocalEnvironment().serialize(),
"core_version": "0.13.0",
}
),
"id": "id",
"name": "name",
}
)
)
assert agent.network == "test-network"
args, kwargs = api.create_container.call_args
assert kwargs["networking_config"] == {"test-network": "config"}
def test_docker_agent_deploy_flow_reg_allow_list_allowed(api):
agent = DockerAgent(reg_allow_list=["test1"])
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"id": "foo",
"storage": Docker(
registry_url="test1", image_name="name", image_tag="tag"
).serialize(),
"environment": LocalEnvironment().serialize(),
"core_version": "0.13.0",
}
),
"id": "id",
"name": "name",
}
)
)
assert api.pull.called
assert api.create_container.called
assert api.start.called
def test_docker_agent_networks_as_modes(api, network):
api.create_networking_config.return_value = {network: "config1"}
api.create_endpoint_config.return_value = "endpoint-config"
agent = DockerAgent(networks=[network])
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"id":
"foo",
"name":
"flow-name",
"storage":
Docker(registry_url="test", image_name="name",
image_tag="tag").serialize(),
"core_version":
"0.13.0",
}),
"id":
"id",
"name":
"name",
}))
assert network in agent.networks
api.create_networking_config.assert_called_once_with(
{network: "endpoint-config"})
_, container_create_kwargs = api.create_container.call_args
assert container_create_kwargs["networking_config"] == {network: "config1"}
_, host_config_kwargs = api.create_host_config.call_args
assert host_config_kwargs["network_mode"] == network
def test_docker_agent_deploy_flow_unsupported_run_config(api):
agent = DockerAgent()
with pytest.raises(
TypeError,
match=
"`run_config` of type `LocalRun`, only `DockerRun` is supported",
):
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"storage": Local().serialize(),
"id": "foo",
"name": "flow-name",
"core_version": "0.13.0",
}),
"run_config":
LocalRun().serialize(),
"id":
"id",
"name":
"name",
"version":
"version",
}))
assert not api.pull.called
def test_docker_agent_deploy_flow_run_config(api, image_on_run_config):
if image_on_run_config:
storage = Local()
image = "on-run-config"
run = DockerRun(image=image, env={"TESTING": "VALUE"})
else:
storage = Docker(
registry_url="testing", image_name="on-storage", image_tag="tag"
)
image = "testing/on-storage:tag"
run = DockerRun(env={"TESTING": "VALUE"})
agent = DockerAgent()
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"id": "foo",
"storage": storage.serialize(),
"run_config": run.serialize(),
"core_version": "0.13.11",
}
),
"id": "id",
"name": "name",
}
)
)
assert api.create_container.called
assert api.create_container.call_args[0][0] == image
assert api.create_container.call_args[1]["environment"]["TESTING"] == "VALUE"
def test_docker_agent_deploy_flow_reg_allow_list_not_allowed(api):
agent = DockerAgent(reg_allow_list=["test1"])
with pytest.raises(ValueError) as error:
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"id": "foo",
"storage": Docker(
registry_url="test2", image_name="name", image_tag="tag"
).serialize(),
"environment": LocalEnvironment().serialize(),
"core_version": "0.13.0",
}
),
"id": "id",
"name": "name",
}
)
)
expected_error = (
"Trying to pull image from a Docker registry 'test2'"
" which is not in the reg_allow_list"
)
assert not api.pull.called
assert not api.create_container.called
assert not api.start.called
assert str(error.value) == expected_error
def test_docker_agent_deploy_flow(core_version, command, api):
agent = DockerAgent()
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"id": "foo",
"storage": Docker(
registry_url="test", image_name="name", image_tag="tag"
).serialize(),
"environment": LocalEnvironment().serialize(),
"core_version": core_version,
}
),
"id": "id",
"name": "name",
}
)
)
assert api.pull.called
assert api.create_container.called
assert api.start.called
assert api.create_host_config.call_args[1]["auto_remove"] is True
assert api.create_container.call_args[1]["command"] == command
assert api.create_container.call_args[1]["host_config"]["AutoRemove"] is True
assert api.start.call_args[1]["container"] == "container_id"
def test_docker_agent_deploy_flow_uses_environment_metadata(api):
agent = DockerAgent()
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"id": "foo",
"storage": Local().serialize(),
"environment": LocalEnvironment(
metadata={"image": "repo/name:tag"}
).serialize(),
"core_version": "0.13.0",
}
),
"id": "id",
"name": "name",
}
)
)
assert api.pull.called
assert api.create_container.called
assert api.start.called
assert api.create_host_config.call_args[1]["auto_remove"] is True
assert api.create_container.call_args[1]["command"] == "prefect execute flow-run"
assert api.create_container.call_args[1]["host_config"]["AutoRemove"] is True
assert api.start.call_args[1]["container"] == "container_id"
def test_docker_agent_deploy_flow_no_registry_does_not_pull(monkeypatch, cloud_api):
api = MagicMock()
api.ping.return_value = True
api.create_container.return_value = {"Id": "container_id"}
monkeypatch.setattr(
"prefect.agent.docker.agent.DockerAgent._get_docker_client",
MagicMock(return_value=api),
)
agent = DockerAgent()
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"id": "foo",
"storage": Docker(
registry_url="", image_name="name", image_tag="tag"
).serialize(),
"environment": LocalEnvironment().serialize(),
"core_version": "0.13.0",
}
),
"id": "id",
"name": "name",
}
)
)
assert not api.pull.called
assert api.create_container.called
assert api.start.called
def test_docker_agent_network(monkeypatch, cloud_api):
api = MagicMock()
api.ping.return_value = True
api.create_container.return_value = {"Id": "container_id"}
api.create_networking_config.return_value = {"test-network": "config"}
monkeypatch.setattr(
"prefect.agent.docker.agent.DockerAgent._get_docker_client",
MagicMock(return_value=api),
)
agent = DockerAgent(network="test-network")
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"id": "foo",
"storage": Docker(
registry_url="test", image_name="name", image_tag="tag"
).serialize(),
"environment": LocalEnvironment().serialize(),
"core_version": "0.13.0",
}
),
"id": "id",
"name": "name",
}
)
)
assert agent.network == "test-network"
args, kwargs = api.create_container.call_args
assert kwargs["networking_config"] == {"test-network": "config"}
def test_docker_agent_networks(api):
api.create_networking_config.return_value = {
"test-network-1": "config1",
"test-network-2": "config2",
}
agent = DockerAgent(networks=["test-network-1", "test-network-2"])
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"id":
"foo",
"name":
"flow-name",
"storage":
Docker(registry_url="test", image_name="name",
image_tag="tag").serialize(),
"core_version":
"0.13.0",
}),
"id":
"id",
"name":
"name",
}))
assert "test-network-1" in agent.networks
assert "test-network-2" in agent.networks
args, kwargs = api.create_container.call_args
assert kwargs["networking_config"] == {
"test-network-1": "config1",
"test-network-2": "config2",
}
def test_docker_agent_networks_no_networks(api):
api.create_networking_config.return_value = {"test-network-1": "config1"}
api.create_endpoint_config.return_value = "endpoint-config"
agent = DockerAgent(networks=[])
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"id":
"foo",
"name":
"flow-name",
"storage":
Docker(registry_url="test", image_name="name",
image_tag="tag").serialize(),
"core_version":
"0.13.0",
}),
"id":
"id",
"name":
"name",
}))
assert agent.networks == []
api.create_networking_config.assert_not_called()
_, kwargs = api.create_container.call_args
assert kwargs["networking_config"] is None
api.connect_container_to_network.assert_not_called()
def test_docker_agent_deploy_flow_show_flow_logs(api, monkeypatch):
process = MagicMock()
monkeypatch.setattr("multiprocessing.Process", process)
agent = DockerAgent(show_flow_logs=True)
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"id": "foo",
"storage": Docker(
registry_url="test", image_name="name", image_tag="tag"
).serialize(),
"environment": LocalEnvironment().serialize(),
"core_version": "0.13.0",
}
),
"id": "id",
"name": "name",
}
)
)
process_kwargs = dict(
target=_stream_container_logs,
kwargs={"base_url": agent.base_url, "container_id": "container_id"},
)
process.assert_called_with(**process_kwargs)
# Check all arguments to `multiprocessing.Process` are pickleable
assert pickle.loads(pickle.dumps(process_kwargs)) == process_kwargs
assert len(agent.processes) == 1
assert api.create_container.called
assert api.start.called
def test_docker_agent_deploy_flow_storage_raises(monkeypatch, api):
monkeypatch.setattr("prefect.agent.agent.Client", MagicMock())
agent = DockerAgent()
with pytest.raises(ValueError):
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"storage": Local().serialize(),
"id": "foo",
"environment": LocalEnvironment().serialize(),
"core_version": "0.13.0",
}
),
"id": "id",
"name": "name",
"version": "version",
}
)
)
assert not api.pull.called
def test_docker_agent_deploy_flow_sets_container_name_with_slugify(
api, run_name, container_name):
"""
Asserts that the container name is set to the flow run name and that collisions with
existing containers with the same name is handled by adding an index
"""
agent = DockerAgent()
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"id":
"foo",
"name":
"flow-name",
"storage":
Local().serialize(),
"environment":
LocalEnvironment(metadata={
"image": "repo/name:tag"
}).serialize(),
"core_version":
"0.13.0",
}),
"id":
"id",
"name":
run_name,
}))
assert api.create_container.call_args[1]["name"] == container_name
def test_docker_agent_deploy_flow_run_config(api, run_kind,
has_docker_storage):
if has_docker_storage:
storage = Docker(registry_url="testing",
image_name="on-storage",
image_tag="tag")
image = "testing/on-storage:tag"
else:
storage = Local()
image = "on-run-config" if run_kind == "docker" else "prefecthq/prefect:0.13.11"
if run_kind == "docker":
env = {"TESTING": "VALUE"}
host_config = {"auto_remove": False, "shm_size": "128m"}
exp_host_config = {
"auto_remove": False,
"extra_hosts": {
"host.docker.internal": "host-gateway"
},
"shm_size": "128m",
}
run = DockerRun(image=image, env=env, host_config=host_config)
else:
env = {}
host_config = {}
exp_host_config = {
"auto_remove": True,
"extra_hosts": {
"host.docker.internal": "host-gateway"
},
}
run = None if run_kind == "missing" else UniversalRun()
agent = DockerAgent()
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"id": "foo",
"name": "flow-name",
"storage": storage.serialize(),
"core_version": "0.13.11",
}),
"run_config":
run.serialize() if run else None,
"id":
"id",
"name":
"name",
}))
assert api.create_container.called
assert api.create_container.call_args[0][0] == image
res_env = api.create_container.call_args[1]["environment"]
for k, v in env.items():
assert res_env[k] == v
res_host_config = api.create_host_config.call_args[1]
for k, v in exp_host_config.items():
assert res_host_config[k] == v
def test_docker_agent_deploy_flow_sets_container_name_with_index(
api, collision_count):
"""
Asserts that the container name is set to the flow run name and that collisions with
existing containers with the same name is handled by adding an index
"""
if collision_count:
# Add the basic name first
existing_names = ["flow-run-name"]
for i in range(1, collision_count):
existing_names.append(f"flow-run-name-{i}")
else:
existing_names = []
def fail_if_name_exists(*args, **kwargs):
if kwargs.get("name") in existing_names:
raise docker.errors.APIError(
"Conflict. The container name 'foobar' is already in use")
return {}
api.create_container = MagicMock(side_effect=fail_if_name_exists)
agent = DockerAgent()
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"id":
"foo",
"name":
"flow-name",
"storage":
Local().serialize(),
"environment":
LocalEnvironment(metadata={
"image": "repo/name:tag"
}).serialize(),
"core_version":
"0.13.0",
}),
"id":
"id",
"name":
"flow-run-name",
}))
expected_name = ("flow-run-name" if not collision_count else
f"flow-run-name-{collision_count}")
assert api.create_container.call_args[1]["name"] == expected_name
def test_docker_agent_deploy_flow_uses_environment_metadata(
monkeypatch, cloud_api):
api = MagicMock()
api.ping.return_value = True
api.create_container.return_value = {"Id": "container_id"}
api.create_host_config.return_value = {"AutoRemove": True}
monkeypatch.setattr(
"prefect.agent.docker.agent.DockerAgent._get_docker_client",
MagicMock(return_value=api),
)
agent = DockerAgent()
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"id":
"foo",
"storage":
Local().serialize(),
"environment":
LocalEnvironment(metadata={
"image": "repo/name:tag"
}).serialize(),
"core_version":
"0.13.0",
}),
"id":
"id",
"name":
"name",
}))
assert api.pull.called
assert api.create_container.called
assert api.start.called
assert api.create_host_config.call_args[1]["auto_remove"] is True
assert api.create_container.call_args[1][
"command"] == "prefect execute flow-run"
assert api.create_container.call_args[1]["host_config"][
"AutoRemove"] is True
assert api.start.call_args[1]["container"] == "container_id"
def test_docker_agent_deploy_flow_run_config(api, run_kind, has_docker_storage):
if has_docker_storage:
storage = Docker(
registry_url="testing", image_name="on-storage", image_tag="tag"
)
image = "testing/on-storage:tag"
else:
storage = Local()
image = "on-run-config" if run_kind == "docker" else "prefecthq/prefect:0.13.11"
if run_kind == "docker":
env = {"TESTING": "VALUE"}
run = DockerRun(image=image, env=env)
else:
env = {}
run = None if run_kind == "missing" else UniversalRun()
agent = DockerAgent()
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"id": "foo",
"name": "flow-name",
"storage": storage.serialize(),
"core_version": "0.13.11",
}
),
"run_config": run.serialize() if run else None,
"id": "id",
"name": "name",
}
)
)
assert api.create_container.called
assert api.create_container.call_args[0][0] == image
res_env = api.create_container.call_args[1]["environment"]
for k, v in env.items():
assert res_env[k] == v
def test_docker_agent_deploy_flow(core_version, command, monkeypatch,
cloud_api):
api = MagicMock()
api.ping.return_value = True
api.create_container.return_value = {"Id": "container_id"}
api.create_host_config.return_value = {"AutoRemove": True}
monkeypatch.setattr(
"prefect.agent.docker.agent.DockerAgent._get_docker_client",
MagicMock(return_value=api),
)
agent = DockerAgent()
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"id":
"foo",
"storage":
Docker(registry_url="test", image_name="name",
image_tag="tag").serialize(),
"environment":
LocalEnvironment().serialize(),
"core_version":
core_version,
}),
"id":
"id",
"name":
"name",
}))
assert api.pull.called
assert api.create_container.called
assert api.start.called
assert api.create_host_config.call_args[1]["auto_remove"] is True
assert api.create_container.call_args[1]["command"] == command
assert api.create_container.call_args[1]["host_config"][
"AutoRemove"] is True
assert api.start.call_args[1]["container"] == "container_id"
def test_docker_agent_deploy_flow_reg_allow_list_not_allowed(monkeypatch, cloud_api):
api = MagicMock()
api.ping.return_value = True
api.create_container.return_value = {"Id": "container_id"}
monkeypatch.setattr(
"prefect.agent.docker.agent.DockerAgent._get_docker_client",
MagicMock(return_value=api),
)
agent = DockerAgent(reg_allow_list=["test1"])
with pytest.raises(ValueError) as error:
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"id": "foo",
"storage": Docker(
registry_url="test2", image_name="name", image_tag="tag"
).serialize(),
"environment": LocalEnvironment().serialize(),
"core_version": "0.13.0",
}
),
"id": "id",
"name": "name",
}
)
)
expected_error = (
"Trying to pull image from a Docker registry 'test2'"
" which is not in the reg_allow_list"
)
assert not api.pull.called
assert not api.create_container.called
assert not api.start.called
assert str(error.value) == expected_error
def test_docker_agent_networks(api):
api.create_networking_config.return_value = {"test-network-1": "config1"}
api.create_endpoint_config.return_value = "endpoint-config"
agent = DockerAgent(networks=["test-network-1", "test-network-2"])
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"id":
"foo",
"name":
"flow-name",
"storage":
Docker(registry_url="test", image_name="name",
image_tag="tag").serialize(),
"core_version":
"0.13.0",
}),
"id":
"id",
"name":
"name",
}))
assert "test-network-1" in agent.networks
assert "test-network-2" in agent.networks
api.create_networking_config.assert_called_once_with(
{"test-network-1": "endpoint-config"})
_, kwargs = api.create_container.call_args
assert kwargs["networking_config"] == {"test-network-1": "config1"}
api.connect_container_to_network.assert_called_once_with(
container=ANY, net_id="test-network-2")
_, host_config_kwargs = api.create_host_config.call_args
assert "network_mode" not in host_config_kwargs
def test_docker_agent_deploy_flow_no_registry_does_not_pull(api):
agent = DockerAgent()
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"id":
"foo",
"name":
"flow-name",
"storage":
Docker(registry_url="", image_name="name",
image_tag="tag").serialize(),
"core_version":
"0.13.0",
}),
"id":
"id",
"name":
"name",
}))
assert not api.pull.called
assert api.create_container.called
assert api.start.called
