def test_local_agent_deploy_no_existing_python_path(monkeypatch, cloud_api):
monkeypatch.delenv("PYTHONPATH", raising=False)
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
agent = LocalAgent(import_paths=["paths"])
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"storage": Local(directory="test").serialize(),
"id": "foo",
"core_version": "0.13.0",
}
),
"id": "id",
}
)
)
assert popen.called
assert len(agent.processes) == 1
assert "paths" in popen.call_args[1]["env"]["PYTHONPATH"]
def test_local_agent_deploy_storage_fails_none(monkeypatch):
client = MagicMock()
set_state = MagicMock()
client.return_value.set_flow_run_state = set_state
monkeypatch.setattr("prefect.agent.agent.Client", client)
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
agent = LocalAgent()
with pytest.raises(ValidationError):
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"storage": None,
"id": "foo",
"core_version": "0.13.0"
}),
"id":
"id",
"version":
1,
}))
assert not popen.called
assert len(agent.processes) == 0
assert client.called
def test_local_agent_deploy_processes_webhook_storage(monkeypatch,
runner_token):
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
agent = LocalAgent()
webhook = Webhook(
build_request_kwargs={"url": "test-service/upload"},
build_request_http_method="POST",
get_flow_request_kwargs={"url": "test-service/download"},
get_flow_request_http_method="GET",
)
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"storage": webhook.serialize(),
"id": "foo",
"core_version": "0.13.0",
}),
"id":
"id",
}))
assert popen.called
assert len(agent.processes) == 1
def test_local_agent_deploy_run_config_missing_working_dir(monkeypatch, tmpdir):
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
working_dir = str(tmpdir.join("missing"))
agent = LocalAgent()
with pytest.raises(ValueError, match="nonexistent `working_dir`"):
agent.deploy_flow(
flow_run=GraphQLResult(
{
"id": "id",
"flow": {
"storage": Local().serialize(),
"id": "foo",
"core_version": "0.13.0",
},
"run_config": LocalRun(working_dir=working_dir).serialize(),
},
)
)
assert not popen.called
assert not agent.processes
def test_local_agent_deploy_storage_raises_not_supported_storage(
monkeypatch, cloud_api
):
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
agent = LocalAgent()
with pytest.raises(ValueError):
agent.deploy_flow(
flow_run=GraphQLResult(
{
"id": "id",
"flow": {
"storage": Docker().serialize(),
"id": "foo",
"core_version": "0.13.0",
},
},
)
)
assert not popen.called
assert len(agent.processes) == 0
def test_local_agent_deploy_run_config_working_dir(monkeypatch, working_dir, tmpdir):
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
if working_dir is not None:
working_dir = str(tmpdir)
agent = LocalAgent()
agent.deploy_flow(
flow_run=GraphQLResult(
{
"id": "id",
"flow": {
"storage": Local().serialize(),
"id": "foo",
"core_version": "0.13.0",
},
"run_config": LocalRun(working_dir=working_dir).serialize(),
},
)
)
assert popen.called
assert len(agent.processes) == 1
assert popen.call_args[1]["cwd"] == working_dir
def test_local_agent_deploy_unsupported_run_config(monkeypatch):
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
agent = LocalAgent()
with pytest.raises(
TypeError,
match="`run_config` of type `KubernetesRun`, only `LocalRun` is supported",
):
agent.deploy_flow(
flow_run=GraphQLResult(
{
"id": "id",
"flow": {
"storage": Local().serialize(),
"id": "foo",
"core_version": "0.13.0",
},
"run_config": KubernetesRun().serialize(),
},
)
)
assert not popen.called
assert len(agent.processes) == 0
def test_local_agent_heartbeat(
monkeypatch, cloud_api, returncode, show_flow_logs, logs
):
popen = MockPopen()
# expect a process to be called with the following command (with specified behavior)
popen.set_command(
"prefect execute flow-run",
stdout=b"awesome output!",
stderr=b"blerg, eRroR!",
returncode=returncode,
poll_count=2,
)
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
agent = LocalAgent(import_paths=["paths"], show_flow_logs=show_flow_logs)
agent.deploy_flow(
flow_run=GraphQLResult(
{
"flow": GraphQLResult(
{
"storage": Local(directory="test").serialize(),
"id": "foo",
"core_version": "0.13.0",
}
),
"id": "id",
}
)
)
process = list(agent.processes)[0]
process_call = process.root_call
with LogCapture() as logcap:
agent.heartbeat()
agent.heartbeat()
agent.heartbeat()
# ensure the expected logs exist (or the absense of logs)
if logs:
logcap.check(*logs)
else:
logcap.check()
# ensure the process was opened and was polled
compare(
popen.all_calls,
expected=[
process_call,
process_call.poll(),
process_call.poll(),
process_call.poll(),
],
)
# the heartbeat should stop tracking upon exit
compare(process.returncode, returncode)
assert len(agent.processes) == 0
def test_local_agent_deploy_processes_s3_storage(monkeypatch, runner_token):
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
agent = LocalAgent()
agent.deploy_flow(flow_run=GraphQLResult(
{
"flow": GraphQLResult({"storage": S3(bucket="test").serialize()}),
"id": "id",
}))
assert popen.called
assert len(agent.processes) == 1
def test_local_agent_deploy_import_paths(monkeypatch, runner_token):
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
agent = LocalAgent(import_paths=["paths"])
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({"storage": Local(directory="test").serialize()}),
"id":
"id",
}))
assert popen.called
assert len(agent.processes) == 1
assert "paths" in popen.call_args[1]["env"]["PYTHONPATH"]
def test_local_agent_deploy_processes_gitlab_storage(monkeypatch, cloud_api):
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
agent = LocalAgent()
gitlab = GitLab("test/repo", path="path/to/flow.py")
agent.deploy_flow(flow_run=GraphQLResult(
{
"flow": GraphQLResult({
"storage": gitlab.serialize(),
"id": "foo",
}),
"id": "id",
}))
assert popen.called
assert len(agent.processes) == 1
def test_local_agent_deploy_storage_raises_not_supported_storage(
monkeypatch, runner_token):
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
agent = LocalAgent()
with pytest.raises(ValueError):
agent.deploy_flow(flow_run=GraphQLResult(
{
"flow": GraphQLResult({"storage": Docker().serialize()}),
"id": "id"
}))
assert not popen.called
assert len(agent.processes) == 0
def test_local_agent_deploy_processes_valid_storage(storage, monkeypatch):
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
agent = LocalAgent()
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"storage": storage.serialize(),
"id": "foo",
"core_version": "0.13.0",
}),
"id":
"id",
}))
assert popen.called
assert len(agent.processes) == 1
def test_local_agent_deploy_processes_local_storage(monkeypatch, cloud_api):
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
agent = LocalAgent()
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"storage": Local(directory="test").serialize(),
"id": "foo"
}),
"id":
"id",
}))
assert popen.called
assert len(agent.processes) == 1
def test_local_agent_deploy_null_or_univeral_run_config(
monkeypatch, run_config):
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
agent = LocalAgent()
agent.deploy_flow(flow_run=GraphQLResult(
{
"id": "id",
"flow": {
"storage": Local().serialize(),
"id": "foo",
"core_version": "0.13.0",
},
"run_config": run_config.serialize() if run_config else None,
}, ))
assert popen.called
assert len(agent.processes) == 1
def test_local_agent_deploy_keep_existing_python_path(monkeypatch, cloud_api):
monkeypatch.setenv("PYTHONPATH", "cool:python:path")
popen = MagicMock()
monkeypatch.setattr("prefect.agent.local.agent.Popen", popen)
agent = LocalAgent(import_paths=["paths"])
agent.deploy_flow(flow_run=GraphQLResult({
"flow":
GraphQLResult({
"storage": Local(directory="test").serialize(),
"id": "foo"
}),
"id":
"id",
}))
python_path = popen.call_args[1]["env"]["PYTHONPATH"]
assert popen.called
assert len(agent.processes) == 1
assert "cool:python:path" in python_path
assert "paths" in python_path
