def test_delete_all_repos():
client = python_pachyderm.experimental.Client()
util.create_test_repo(client, "test_delete_all_repos", prefix="extra-1")
util.create_test_repo(client, "test_delete_all_repos", prefix="extra-2")
assert len(list(client.list_repo())) >= 2
client.delete_all_repos()
assert len(list(client.list_repo())) == 0
def test_create_pipeline_from_request():
client = python_pachyderm.experimental.Client()
repo_name = util.create_test_repo(client,
"test_create_pipeline_from_request")
pipeline_name = util.test_repo_name("test_create_pipeline_from_request")
# more or less a copy of the opencv demo's edges pipeline spec
client.create_pipeline_from_request(
pps_proto.CreatePipelineRequest(
pipeline=pps_proto.Pipeline(name=pipeline_name),
description=
"A pipeline that performs image edge detection by using the OpenCV library.",
input=pps_proto.Input(pfs=pps_proto.PfsInput(
glob="/*",
repo=repo_name,
), ),
transform=pps_proto.Transform(
cmd=["echo", "hi"],
image="pachyderm/opencv",
),
))
assert any(p.pipeline.name == pipeline_name
for p in list(client.list_pipeline()))
def test_list_commit():
python_pachyderm.experimental.Client().delete_all_repos()
client, repo_name1 = sandbox("list_commit1")
with client.commit(repo_name1, "master"):
pass
with client.commit(repo_name1, "master"):
pass
repo_name2 = util.create_test_repo(client, "list_commit2")
with client.commit(repo_name2, "master"):
pass
commits = list(client.list_commit())
assert len(commits) == 3
def test_create_pipeline():
client = python_pachyderm.experimental.Client()
client.delete_all()
input_repo_name = util.create_test_repo(client,
"input_repo_test_create_pipeline")
client.create_pipeline(
"pipeline_test_create_pipeline",
transform=pps_proto.Transform(
cmd=["sh"],
image="alpine",
stdin=["cp /pfs/{}/*.dat /pfs/out/".format(input_repo_name)],
),
input=pps_proto.Input(
pfs=pps_proto.PfsInput(glob="/*", repo=input_repo_name)),
)
assert len(list(client.list_pipeline())) == 1
def test_drop_commit():
client, repo_name = sandbox("drop_commit")
repo2_name = util.create_test_repo(client, "drop_commit2")
# Create provenance between repos (which creates an auto commit)
client.create_branch(
repo2_name,
"master",
provenance=[
pfs_proto.Branch(
repo=pfs_proto.Repo(name=repo_name, type="user"), name="master"
)
],
)
# Head commit is still open in repo2
client.finish_commit((repo2_name, "master"))
with client.commit(repo_name, "master"):
pass
client.finish_commit((repo2_name, "master"))
with client.commit(repo_name, "master") as commit2:
pass
client.finish_commit((repo2_name, "master"))
client.wait_commit(commit2.id)
commits = list(client.list_commit(repo_name))
assert len(commits) == 2
client.drop_commit(commit2.id)
commits = list(client.list_commit(repo_name))
assert len(commits) == 1
commits = list(client.list_commit(repo2_name))
assert len(commits) == 0 # since list_commit defaults to user commits
commits = list(
client.list_commit(repo2_name, origin_kind=pfs_proto.OriginKind.AUTO)
)
assert len(commits) == 2
def test_wait_commit():
"""
Ensure wait_commit works
"""
client, repo_name = sandbox("wait_commit")
repo2_name = util.create_test_repo(client, "wait_commit2")
# Create provenance between repos (which creates a new commit)
client.create_branch(
repo2_name,
"master",
provenance=[
pfs_proto.Branch(
repo=pfs_proto.Repo(name=repo_name, type="user"), name="master"
)
],
)
# Head commit is still open in repo2
client.finish_commit((repo2_name, "master"))
with client.commit(repo_name, "master") as c:
client.put_file_bytes(c, "input.json", b"hello world")
client.finish_commit((repo2_name, "master"))
# Just block until all of the commits are yielded
commits = client.wait_commit(c.id)
assert len(commits) == 2
assert commits[1].finished
with client.commit(repo_name, "master") as c2:
client.put_file_bytes(c2, "input.json", b"bye world")
client.finish_commit((repo2_name, "master"))
# Just block until the commit in repo1 is finished
commits = client.wait_commit(c2)
assert len(commits) == 1
assert commits[0].finished
files = list(client.list_file(c2, "/"))
assert len(files) == 1
def test_mount():
client, repo_name = sandbox("mount")
repo2_name = util.create_test_repo(client, "mount2")
with client.commit(repo_name, "master") as c:
client.put_file_bytes(c, "/file1.txt", b"DATA1")
with client.commit(repo2_name, "master") as c2:
client.put_file_bytes(c2, "/file2.txt", b"DATA2")
# Mount all repos
client.mount("mount_a")
assert open(f"mount_a/{repo_name}/file1.txt").read() == "DATA1"
assert open(f"mount_a/{repo2_name}/file2.txt").read() == "DATA2"
# Mount one repo
client.mount("mount_b", [repo2_name])
assert open(f"mount_b/{repo2_name}/file2.txt").read() == "DATA2"
client.unmount(all_mounts=True)
assert not os.path.exists(f"mount_a/{repo2_name}/file2.txt")
assert not os.path.exists(f"mount_b/{repo2_name}/file2.txt")
with client.commit(repo_name, "dev") as c3:
client.put_file_bytes(c3, "/file3.txt", b"DATA3")
# Mount one repo
client.mount("mount_c", [f"{repo_name}@dev"])
assert open(f"mount_c/{repo_name}/file3.txt").read() == "DATA3"
client.unmount("mount_c")
assert not os.path.exists(f"mount_c/{repo_name}/file3.txt")
# Test runtime error
Path("mount_d").mkdir()
Path("mount_d/file.txt").touch()
with pytest.raises(RuntimeError, match="must be empty to mount"):
client.mount("mount_d")
def test_inspect_commit():
client, repo_name = sandbox("inspect_commit")
repo2_name = util.create_test_repo(client, "inspect_commit2")
# Create provenance between repos (which creates a new commit)
client.create_branch(
repo2_name,
"master",
provenance=[
pfs_proto.Branch(
repo=pfs_proto.Repo(name=repo_name, type="user"), name="master"
)
],
)
# Head commit is still open in repo2
client.finish_commit((repo2_name, "master"))
with client.commit(repo_name, "master") as c:
client.put_file_bytes(c, "input.json", b"hello world")
client.finish_commit((repo2_name, "master"))
# Inspect commit at a specific repo
commits = list(client.inspect_commit(c, pfs_proto.CommitState.FINISHED))
assert len(commits) == 1
commit = commits[0]
assert commit.commit.branch.name == "master"
assert commit.finished
assert commit.description == ""
# assert commit.size_bytes == 11
assert len(commit.commit.id) == 32
assert commit.commit.branch.repo.name == repo_name
# Inspect entire commit
commits = list(client.inspect_commit(c.id, pfs_proto.CommitState.FINISHED))
assert len(commits) == 2
def sandbox(test_name):
client = python_pachyderm.experimental.Client()
repo_name = util.create_test_repo(client, test_name)
return client, repo_name