async def test_volumes(live_config_loader: ConfigLoader) -> None:
flow = await RunningLiveFlow.create(live_config_loader, "live-full")
ctx = flow._ctx
assert ctx.volumes.keys() == {"volume_a", "volume_b"}
assert ctx.volumes["volume_a"].id == "volume_a"
assert ctx.volumes["volume_a"].remote == URL("storage:dir")
assert ctx.volumes["volume_a"].mount == RemotePath("/var/dir")
assert ctx.volumes["volume_a"].read_only
assert ctx.volumes["volume_a"].local == LocalPath("dir")
assert (ctx.volumes["volume_a"].full_local_path ==
live_config_loader.workspace / "dir")
assert ctx.volumes["volume_a"].ref_ro == "storage:dir:/var/dir:ro"
assert ctx.volumes["volume_a"].ref_rw == "storage:dir:/var/dir:rw"
assert ctx.volumes["volume_a"].ref == "storage:dir:/var/dir:ro"
assert ctx.volumes["volume_b"].id == "volume_b"
assert ctx.volumes["volume_b"].remote == URL("storage:other")
assert ctx.volumes["volume_b"].mount == RemotePath("/var/other")
assert not ctx.volumes["volume_b"].read_only
assert ctx.volumes["volume_b"].local is None
assert ctx.volumes["volume_b"].full_local_path is None
assert ctx.volumes["volume_b"].ref_ro == "storage:other:/var/other:ro"
assert ctx.volumes["volume_b"].ref_rw == "storage:other:/var/other:rw"
assert ctx.volumes["volume_b"].ref == "storage:other:/var/other:rw"
async def test_pipeline_minimal_ctx(batch_config_loader: ConfigLoader) -> None:
flow = await RunningBatchFlow.create(batch_config_loader, "batch-minimal",
"bake-id")
task = await flow.get_task((), "test_a", needs={}, state={})
assert task.id == "test_a"
assert task.title == "Batch title"
assert task.name == "job-name"
assert task.image == "image:banana"
assert task.preset == "cpu-micro"
assert task.http_port == 8080
assert not task.http_auth
assert task.entrypoint == "bash"
assert task.cmd == "echo abc"
assert task.workdir == RemotePath("/local/dir")
assert task.volumes == [
"storage:common:/mnt/common:rw",
"storage:dir:/var/dir:ro",
"storage:dir:/var/dir:ro",
]
assert task.tags == {
"tag-1",
"tag-2",
"tag-a",
"tag-b",
"task:test-a",
"project:unit",
"flow:batch-minimal",
"bake_id:bake-id",
}
assert task.life_span == 10500.0
assert task.strategy.max_parallel == 10
assert task.strategy.fail_fast
assert flow.graph == {"test_a": {}}
async def test_job(live_config_loader: ConfigLoader) -> None:
flow = await RunningLiveFlow.create(live_config_loader, "live-full")
job = await flow.get_job("test_a", {})
assert job.id == "test_a"
assert job.title == "Job title"
assert job.name == "job-name"
assert job.image == "image:banana"
assert job.preset == "cpu-micro"
assert job.http_port == 8080
assert not job.http_auth
assert job.entrypoint == "bash"
assert job.cmd == "echo abc"
assert job.workdir == RemotePath("/local/dir")
assert job.volumes == [
"storage:common:/mnt/common:rw",
"storage:dir:/var/dir:ro",
"storage:dir:/var/dir:ro",
]
assert job.tags == {
"tag-1",
"tag-2",
"tag-a",
"tag-b",
"project:unit",
"flow:live-full",
"job:test-a",
}
assert job.life_span == 10500.0
assert job.port_forward == ["2211:22"]
assert job.detach
assert job.browse
async def test_project_level_defaults_live(assets: pathlib.Path,
client: Client) -> None:
ws = assets / "with_project_yaml"
config_dir = ConfigDir(
workspace=ws,
config_dir=ws,
)
cl = LiveLocalCL(config_dir, client)
try:
flow = await RunningLiveFlow.create(cl, "live")
job = await flow.get_job("test", {})
assert "tag-a" in job.tags
assert "tag-b" in job.tags
assert job.env["global_a"] == "val-a"
assert job.env["global_b"] == "val-b"
assert job.env["global_b"] == "val-b"
assert job.volumes == [
"storage:common:/mnt/common:rw",
"storage:dir:/var/dir:ro",
]
assert job.workdir == RemotePath("/global/dir")
assert job.life_span == 100800.0
assert job.preset == "cpu-large"
assert job.schedule_timeout == 2157741.0
assert job.image == "image:banana"
finally:
await cl.close()
async def test_batch_with_project_globals(assets: pathlib.Path,
client: Client) -> None:
ws = assets / "with_project_yaml"
config_dir = ConfigDir(
workspace=ws,
config_dir=ws,
)
cl = BatchLocalCL(config_dir, client)
try:
flow = await RunningBatchFlow.create(cl, "batch", "bake-id")
task = await flow.get_task((), "task", needs={}, state={})
assert "tag-a" in task.tags
assert "tag-b" in task.tags
assert task.env["global_a"] == "val-a"
assert task.env["global_b"] == "val-b"
assert task.volumes == [
"storage:common:/mnt/common:rw",
"storage:dir:/var/dir:ro",
]
assert task.workdir == RemotePath("/global/dir")
assert task.life_span == 100800.0
assert task.preset == "cpu-large"
assert task.schedule_timeout == 2157741.0
assert task.image == "image:main"
assert not task.strategy.fail_fast
assert task.strategy.max_parallel == 20
assert task.cache.strategy == CacheStrategy.NONE
assert task.cache.life_span == 9000.0
finally:
await cl.close()
async def test_defaults(live_config_loader: ConfigLoader) -> None:
flow = await RunningLiveFlow.create(live_config_loader, "live-full")
ctx = flow._ctx
assert ctx.tags == {"tag-a", "tag-b", "project:unit", "flow:live-full"}
assert flow._defaults.workdir == RemotePath("/global/dir")
assert flow._defaults.life_span == 100800.0
assert flow._defaults.preset == "cpu-large"
assert flow._defaults.schedule_timeout == 2157741.0
async def test_local_remote_path_images(
client: Client, live_config_loader: ConfigLoader) -> None:
flow = await RunningLiveFlow.create(live_config_loader,
"live-different-images")
ctx = flow._ctx
assert ctx.images.keys() == {"image_local", "image_remote"}
assert ctx.images[
"image_local"].context == live_config_loader.workspace / "dir"
assert (
ctx.images["image_local"].dockerfile == live_config_loader.workspace /
"dir/Dockerfile")
assert ctx.images["image_local"].dockerfile_rel == LocalPath("Dockerfile")
assert ctx.images["image_remote"].context == URL(
f"storage://{client.cluster_name}/{client.username}/dir")
assert ctx.images["image_remote"].dockerfile == URL(
f"storage://{client.cluster_name}/{client.username}/dir/Dockerfile")
assert ctx.images["image_remote"].dockerfile_rel == RemotePath(
"Dockerfile")
async def test_upload_dry_run_mode_prints_commands(
client: Client,
live_context_factory: LiveContextFactory,
capsys: CaptureFixture[str],
) -> None:
expr = StrExpr(
POS,
POS,
"${{ upload(volumes.test) }}",
)
ctx = live_context_factory(
client,
set(),
dry_run=True,
volumes={
"test":
VolumeCtx(
id="test",
full_local_path=LocalPath("/test/local"),
local=LocalPath("local"),
remote=URL("storage://cluster/user/somedir"),
read_only=False,
mount=RemotePath("/mnt"),
)
},
)
await expr.eval(ctx)
capture = capsys.readouterr()
assert "neuro mkdir --parents storage://cluster/user\n" in capture.out
if sys.platform == "win32":
assert ("neuro cp --recursive --update --no-target-directory"
" '\\test\\local' storage://cluster/user/somedir\n"
in capture.out)
else:
assert ("neuro cp --recursive --update --no-target-directory"
" /test/local storage://cluster/user/somedir\n" in capture.out)