def test_delete_all_repos():
client = python_pachyderm.Client()
util.create_test_repo(client, "test_delete_all_repos", prefix="extra-1")
util.create_test_repo(client, "test_delete_all_repos", prefix="extra-2")
assert len(list(client.list_repo())) >= 2
client.delete_all_repos()
assert len(list(client.list_repo())) == 0
def test_transaction_context_mgr_exception():
client = python_pachyderm.experimental.Client()
expected_repo_count = len(list(client.list_repo()))
with pytest.raises(Exception):
with client.transaction():
util.create_test_repo(client, "test_transaction_context_mgr_exception")
util.create_test_repo(client, "test_transaction_context_mgr_exception")
raise Exception("oops!")
assert len(client.list_transaction()) == 0
assert len(list(client.list_repo())) == expected_repo_count
def test_inspect_commit():
client, repo_name = sandbox("inspect_commit")
repo2_name = util.create_test_repo(client, "inspect_commit2")
# Create provenance between repos (which creates a new commit)
client.create_branch(
repo2_name,
"master",
provenance=[
pfs_proto.Branch(repo=pfs_proto.Repo(name=repo_name, type="user"),
name="master")
],
)
# Head commit is still open in repo2
client.finish_commit((repo2_name, "master"))
with client.commit(repo_name, "master") as c:
client.put_file_bytes(c, "input.json", b"hello world")
client.finish_commit((repo2_name, "master"))
# Inspect commit at a specific repo
commits = list(client.inspect_commit(c, pfs_proto.CommitState.FINISHED))
assert len(commits) == 1
commit = commits[0]
assert commit.commit.branch.name == "master"
assert commit.finished
assert commit.description == ""
# assert commit.size_bytes == 11
assert len(commit.commit.id) == 32
assert commit.commit.branch.repo.name == repo_name
# Inspect entire commit
commits = list(client.inspect_commit(c.id, pfs_proto.CommitState.FINISHED))
assert len(commits) == 2
def test_put_files():
client = python_pachyderm.Client()
repo_name = util.create_test_repo(client, "put_files")
with tempfile.TemporaryDirectory(suffix="python_pachyderm") as d:
# create a temporary directory with these files:
# 0.txt 1.txt 2.txt 3.txt 4.txt 0/0.txt 1/1.txt 2/2.txt
# 3/3.txt 4/4.txt
for i in range(5):
os.makedirs(os.path.join(d, str(i)))
for j in range(5):
with open(os.path.join(d, "{}.txt".format(j)), "w") as f:
f.write(str(j))
with open(os.path.join(d, str(j), "{}.txt".format(j)), "w") as f:
f.write(str(j))
# add the files under both `/` and `/sub` (the latter redundantly to
# test both for correct path handling and the ability to put files
# that already exist)
commit = "{}/master".format(repo_name)
python_pachyderm.put_files(client, d, commit, "/")
python_pachyderm.put_files(client, d, commit, "/sub")
python_pachyderm.put_files(client, d, commit, "/sub/")
expected = set(["/", "/sub"])
for i in range(5):
expected.add("/{}".format(i))
expected.add("/{}.txt".format(i))
expected.add("/{}/{}.txt".format(i, i))
expected.add("/sub/{}".format(i))
expected.add("/sub/{}.txt".format(i))
expected.add("/sub/{}/{}.txt".format(i, i))
check_expected_files(client, commit, expected)
def test_create_pipeline_from_request():
client = python_pachyderm.Client()
repo_name = util.create_test_repo(client, "test_create_pipeline_from_request")
pipeline_name = util.test_repo_name("test_create_pipeline_from_request")
# more or less a copy of the opencv demo's edges pipeline spec
client.create_pipeline_from_request(
pps_proto.CreatePipelineRequest(
pipeline=pps_proto.Pipeline(name=pipeline_name),
description="A pipeline that performs image edge detection by using the OpenCV library.",
input=pps_proto.Input(
pfs=pps_proto.PFSInput(
glob="/*",
repo=repo_name,
),
),
transform=pps_proto.Transform(
cmd=["echo", "hi"],
image="pachyderm/opencv",
),
)
)
assert any(p.pipeline.name == pipeline_name for p in list(client.list_pipeline()))
def test_delete_transaction():
client = python_pachyderm.Client()
expected_repo_count = len(list(client.list_repo()))
transaction = client.start_transaction()
util.create_test_repo(client, "test_delete_transaction")
util.create_test_repo(client, "test_delete_transaction")
client.delete_transaction(transaction)
assert len(client.list_transaction()) == 0
# even though the transaction was deleted, the repos were still created,
# because the transaction wasn't tied to the client
assert len(list(client.list_repo())) == expected_repo_count + 2
with pytest.raises(python_pachyderm.RpcError):
# re-deleting should cause an error
client.delete_transaction(transaction)
def test_scope():
with sandbox() as client:
repo = util.create_test_repo(client, "test_scope")
scopes = client.get_scope("robot:root", repo)
assert all(s == python_pachyderm.Scope.NONE.value for s in scopes)
client.set_scope("robot:root", repo, python_pachyderm.Scope.READER.value)
scopes = client.get_scope("robot:root", repo)
assert all(s == python_pachyderm.Scope.NONE.value for s in scopes)
def test_transaction_context_mgr():
client = python_pachyderm.Client()
expected_repo_count = len(list(client.list_repo())) + 2
with client.transaction() as transaction:
util.create_test_repo(client, "test_transaction_context_mgr")
util.create_test_repo(client, "test_transaction_context_mgr")
transactions = client.list_transaction()
assert len(transactions) == 1
assert transactions[0].transaction.id == transaction.id
assert client.inspect_transaction(
transaction).transaction.id == transaction.id
assert (client.inspect_transaction(
transaction.id).transaction.id == transaction.id)
assert len(client.list_transaction()) == 0
assert len(list(client.list_repo())) == expected_repo_count
def test_put_files_single_file():
client = python_pachyderm.experimental.Client()
client.delete_all()
repo_name = util.create_test_repo(client, "put_files_single_file")
with tempfile.NamedTemporaryFile() as f:
f.write(b"abcd")
f.flush()
commit = (repo_name, "master")
python_pachyderm.put_files(client, f.name, commit, "/f1.txt")
python_pachyderm.put_files(client, f.name, commit, "/f/f1")
expected = set(["/", "/f1.txt", "/f/", "/f/f1"])
check_expected_files(client, commit, expected)
def test_create_python_pipeline_bad_path():
client = python_pachyderm.Client()
repo_name = util.create_test_repo(client, "create_python_pipeline_bad_path")
# create some sample data
with client.commit(repo_name, "master") as commit:
client.put_file_bytes(commit, 'file.dat', b'DATA')
# create a pipeline from a file that does not exist - should fail
with pytest.raises(Exception):
python_pachyderm.create_python_pipeline(
client, "./foobar2000",
input=python_pachyderm.Input(pfs=python_pachyderm.PFSInput(glob="/", repo=repo_name)),
)
def test_acl():
def verify_acl(client, repo):
acl = client.get_acl(repo)
assert len(acl.entries) == 1
assert len(acl.robot_entries) == 0
assert acl.entries[0].username == "robot:root"
assert acl.entries[0].scope == python_pachyderm.Scope.OWNER.value
return acl
with sandbox() as client:
repo = util.create_test_repo(client, "test_acl")
acl = verify_acl(client, repo)
client.set_acl(repo, acl.entries)
verify_acl(client, repo)
def test_create_pipeline():
client = python_pachyderm.Client()
client.delete_all()
input_repo_name = util.create_test_repo(client, "input_repo_test_create_pipeline")
client.create_pipeline(
"pipeline_test_create_pipeline",
transform=pps_proto.Transform(
cmd=["sh"],
image="alpine",
stdin=["cp /pfs/{}/*.dat /pfs/out/".format(input_repo_name)],
),
input=pps_proto.Input(pfs=pps_proto.PFSInput(glob="/*", repo=input_repo_name)),
)
assert len(list(client.list_pipeline())) == 1
def test_list_commit():
python_pachyderm.Client().delete_all_repos()
client, repo_name1 = sandbox("list_commit1")
with client.commit(repo_name1, "master"):
pass
with client.commit(repo_name1, "master"):
pass
repo_name2 = util.create_test_repo(client, "list_commit2")
with client.commit(repo_name2, "master"):
pass
commits = list(client.list_commit())
assert len(commits) == 3
def test_wait_commit():
"""
Ensure wait_commit works
"""
client, repo_name = sandbox("wait_commit")
repo2_name = util.create_test_repo(client, "wait_commit2")
# Create provenance between repos (which creates a new commit)
client.create_branch(
repo2_name,
"master",
provenance=[
pfs_proto.Branch(repo=pfs_proto.Repo(name=repo_name, type="user"),
name="master")
],
)
# Head commit is still open in repo2
client.finish_commit((repo2_name, "master"))
with client.commit(repo_name, "master") as c:
client.put_file_bytes(c, "input.json", b"hello world")
client.finish_commit((repo2_name, "master"))
# Just block until all of the commits are yielded
commits = client.wait_commit(c.id)
assert len(commits) == 2
assert commits[1].finished
with client.commit(repo_name, "master") as c2:
client.put_file_bytes(c2, "input.json", b"bye world")
client.finish_commit((repo2_name, "master"))
# Just block until the commit in repo1 is finished
commits = client.wait_commit(c2)
assert len(commits) == 1
assert commits[0].finished
files = list(client.list_file(c2, "/"))
assert len(files) == 1
def test_drop_commit():
client, repo_name = sandbox("drop_commit")
repo2_name = util.create_test_repo(client, "drop_commit2")
# Create provenance between repos (which creates an auto commit)
client.create_branch(
repo2_name,
"master",
provenance=[
pfs_proto.Branch(repo=pfs_proto.Repo(name=repo_name, type="user"),
name="master")
],
)
# Head commit is still open in repo2
client.finish_commit((repo2_name, "master"))
with client.commit(repo_name, "master"):
pass
client.finish_commit((repo2_name, "master"))
with client.commit(repo_name, "master") as commit2:
pass
client.finish_commit((repo2_name, "master"))
client.wait_commit(commit2.id)
commits = list(client.list_commit(repo_name))
assert len(commits) == 2
client.drop_commit(commit2.id)
commits = list(client.list_commit(repo_name))
assert len(commits) == 1
commits = list(client.list_commit(repo2_name))
assert len(commits) == 0 # since list_commit defaults to user commits
commits = list(
client.list_commit(repo2_name, origin_kind=pfs_proto.OriginKind.AUTO))
assert len(commits) == 2
def sandbox(test_name):
client = python_pachyderm.Client()
repo_name = util.create_test_repo(client, test_name)
return client, repo_name
def test_create_python_pipeline():
client = python_pachyderm.Client()
repo_name = util.create_test_repo(client, "create_python_pipeline")
pfs_input = python_pachyderm.Input(pfs=python_pachyderm.PFSInput(glob="/", repo=repo_name))
pipeline_name = util.test_repo_name("create_python_pipeline", prefix="pipeline")
# create some sample data
with client.commit(repo_name, "master") as commit:
client.put_file_bytes(commit, 'file.dat', b'DATA')
# convenience function for verifying expected files exist
def check_all_expected_files(extra_source_files, extra_build_files):
list(client.flush_commit([c.commit for c in client.list_commit(pipeline_name)]))
check_expected_files(client, "{}_build/source".format(pipeline_name), set([
"/",
"/main.py",
*extra_source_files,
]))
check_expected_files(client, "{}_build/build".format(pipeline_name), set([
"/",
"/run.sh",
*extra_build_files,
]))
check_expected_files(client, "{}/master".format(pipeline_name), set([
"/",
"/file.dat",
]))
# 1) create a pipeline from a directory with a main.py and requirements.txt
with tempfile.TemporaryDirectory(suffix="python_pachyderm") as d:
with open(os.path.join(d, "main.py"), "w") as f:
f.write(TEST_LIB_SOURCE.format(repo_name))
with open(os.path.join(d, "requirements.txt"), "w") as f:
f.write(TEST_REQUIREMENTS_SOURCE)
python_pachyderm.create_python_pipeline(
client, d,
input=pfs_input,
pipeline_name=pipeline_name,
)
check_all_expected_files(
["/requirements.txt"],
["/leftpad-0.1.2-py3-none-any.whl", "/termcolor-1.1.0-py3-none-any.whl"],
)
file = list(client.get_file('{}/master'.format(pipeline_name), 'file.dat'))
assert file == [b' DATA']
# 2) update pipeline from a directory without a requirements.txt
with tempfile.TemporaryDirectory(suffix="python_pachyderm") as d:
with open(os.path.join(d, "main.py"), "w") as f:
f.write(TEST_STDLIB_SOURCE.format(repo_name))
python_pachyderm.create_python_pipeline(
client, d,
input=pfs_input,
pipeline_name=pipeline_name,
update=True,
)
check_all_expected_files([], [])
file = list(client.get_file('{}/master'.format(pipeline_name), 'file.dat'))
assert file == [b'DATA']
