def test_delete_all_repos(): client = python_pachyderm.Client() util.create_test_repo(client, "test_delete_all_repos", prefix="extra-1") util.create_test_repo(client, "test_delete_all_repos", prefix="extra-2") assert len(list(client.list_repo())) >= 2 client.delete_all_repos() assert len(list(client.list_repo())) == 0
def test_transaction_context_mgr_exception(): client = python_pachyderm.experimental.Client() expected_repo_count = len(list(client.list_repo())) with pytest.raises(Exception): with client.transaction(): util.create_test_repo(client, "test_transaction_context_mgr_exception") util.create_test_repo(client, "test_transaction_context_mgr_exception") raise Exception("oops!") assert len(client.list_transaction()) == 0 assert len(list(client.list_repo())) == expected_repo_count
def test_inspect_commit(): client, repo_name = sandbox("inspect_commit") repo2_name = util.create_test_repo(client, "inspect_commit2") # Create provenance between repos (which creates a new commit) client.create_branch( repo2_name, "master", provenance=[ pfs_proto.Branch(repo=pfs_proto.Repo(name=repo_name, type="user"), name="master") ], ) # Head commit is still open in repo2 client.finish_commit((repo2_name, "master")) with client.commit(repo_name, "master") as c: client.put_file_bytes(c, "input.json", b"hello world") client.finish_commit((repo2_name, "master")) # Inspect commit at a specific repo commits = list(client.inspect_commit(c, pfs_proto.CommitState.FINISHED)) assert len(commits) == 1 commit = commits[0] assert commit.commit.branch.name == "master" assert commit.finished assert commit.description == "" # assert commit.size_bytes == 11 assert len(commit.commit.id) == 32 assert commit.commit.branch.repo.name == repo_name # Inspect entire commit commits = list(client.inspect_commit(c.id, pfs_proto.CommitState.FINISHED)) assert len(commits) == 2
def test_put_files(): client = python_pachyderm.Client() repo_name = util.create_test_repo(client, "put_files") with tempfile.TemporaryDirectory(suffix="python_pachyderm") as d: # create a temporary directory with these files: # 0.txt 1.txt 2.txt 3.txt 4.txt 0/0.txt 1/1.txt 2/2.txt # 3/3.txt 4/4.txt for i in range(5): os.makedirs(os.path.join(d, str(i))) for j in range(5): with open(os.path.join(d, "{}.txt".format(j)), "w") as f: f.write(str(j)) with open(os.path.join(d, str(j), "{}.txt".format(j)), "w") as f: f.write(str(j)) # add the files under both `/` and `/sub` (the latter redundantly to # test both for correct path handling and the ability to put files # that already exist) commit = "{}/master".format(repo_name) python_pachyderm.put_files(client, d, commit, "/") python_pachyderm.put_files(client, d, commit, "/sub") python_pachyderm.put_files(client, d, commit, "/sub/") expected = set(["/", "/sub"]) for i in range(5): expected.add("/{}".format(i)) expected.add("/{}.txt".format(i)) expected.add("/{}/{}.txt".format(i, i)) expected.add("/sub/{}".format(i)) expected.add("/sub/{}.txt".format(i)) expected.add("/sub/{}/{}.txt".format(i, i)) check_expected_files(client, commit, expected)
def test_create_pipeline_from_request(): client = python_pachyderm.Client() repo_name = util.create_test_repo(client, "test_create_pipeline_from_request") pipeline_name = util.test_repo_name("test_create_pipeline_from_request") # more or less a copy of the opencv demo's edges pipeline spec client.create_pipeline_from_request( pps_proto.CreatePipelineRequest( pipeline=pps_proto.Pipeline(name=pipeline_name), description="A pipeline that performs image edge detection by using the OpenCV library.", input=pps_proto.Input( pfs=pps_proto.PFSInput( glob="/*", repo=repo_name, ), ), transform=pps_proto.Transform( cmd=["echo", "hi"], image="pachyderm/opencv", ), ) ) assert any(p.pipeline.name == pipeline_name for p in list(client.list_pipeline()))
def test_delete_transaction(): client = python_pachyderm.Client() expected_repo_count = len(list(client.list_repo())) transaction = client.start_transaction() util.create_test_repo(client, "test_delete_transaction") util.create_test_repo(client, "test_delete_transaction") client.delete_transaction(transaction) assert len(client.list_transaction()) == 0 # even though the transaction was deleted, the repos were still created, # because the transaction wasn't tied to the client assert len(list(client.list_repo())) == expected_repo_count + 2 with pytest.raises(python_pachyderm.RpcError): # re-deleting should cause an error client.delete_transaction(transaction)
def test_scope(): with sandbox() as client: repo = util.create_test_repo(client, "test_scope") scopes = client.get_scope("robot:root", repo) assert all(s == python_pachyderm.Scope.NONE.value for s in scopes) client.set_scope("robot:root", repo, python_pachyderm.Scope.READER.value) scopes = client.get_scope("robot:root", repo) assert all(s == python_pachyderm.Scope.NONE.value for s in scopes)
def test_transaction_context_mgr(): client = python_pachyderm.Client() expected_repo_count = len(list(client.list_repo())) + 2 with client.transaction() as transaction: util.create_test_repo(client, "test_transaction_context_mgr") util.create_test_repo(client, "test_transaction_context_mgr") transactions = client.list_transaction() assert len(transactions) == 1 assert transactions[0].transaction.id == transaction.id assert client.inspect_transaction( transaction).transaction.id == transaction.id assert (client.inspect_transaction( transaction.id).transaction.id == transaction.id) assert len(client.list_transaction()) == 0 assert len(list(client.list_repo())) == expected_repo_count
def test_put_files_single_file(): client = python_pachyderm.experimental.Client() client.delete_all() repo_name = util.create_test_repo(client, "put_files_single_file") with tempfile.NamedTemporaryFile() as f: f.write(b"abcd") f.flush() commit = (repo_name, "master") python_pachyderm.put_files(client, f.name, commit, "/f1.txt") python_pachyderm.put_files(client, f.name, commit, "/f/f1") expected = set(["/", "/f1.txt", "/f/", "/f/f1"]) check_expected_files(client, commit, expected)
def test_create_python_pipeline_bad_path(): client = python_pachyderm.Client() repo_name = util.create_test_repo(client, "create_python_pipeline_bad_path") # create some sample data with client.commit(repo_name, "master") as commit: client.put_file_bytes(commit, 'file.dat', b'DATA') # create a pipeline from a file that does not exist - should fail with pytest.raises(Exception): python_pachyderm.create_python_pipeline( client, "./foobar2000", input=python_pachyderm.Input(pfs=python_pachyderm.PFSInput(glob="/", repo=repo_name)), )
def test_acl(): def verify_acl(client, repo): acl = client.get_acl(repo) assert len(acl.entries) == 1 assert len(acl.robot_entries) == 0 assert acl.entries[0].username == "robot:root" assert acl.entries[0].scope == python_pachyderm.Scope.OWNER.value return acl with sandbox() as client: repo = util.create_test_repo(client, "test_acl") acl = verify_acl(client, repo) client.set_acl(repo, acl.entries) verify_acl(client, repo)
def test_create_pipeline(): client = python_pachyderm.Client() client.delete_all() input_repo_name = util.create_test_repo(client, "input_repo_test_create_pipeline") client.create_pipeline( "pipeline_test_create_pipeline", transform=pps_proto.Transform( cmd=["sh"], image="alpine", stdin=["cp /pfs/{}/*.dat /pfs/out/".format(input_repo_name)], ), input=pps_proto.Input(pfs=pps_proto.PFSInput(glob="/*", repo=input_repo_name)), ) assert len(list(client.list_pipeline())) == 1
def test_list_commit(): python_pachyderm.Client().delete_all_repos() client, repo_name1 = sandbox("list_commit1") with client.commit(repo_name1, "master"): pass with client.commit(repo_name1, "master"): pass repo_name2 = util.create_test_repo(client, "list_commit2") with client.commit(repo_name2, "master"): pass commits = list(client.list_commit()) assert len(commits) == 3
def test_wait_commit(): """ Ensure wait_commit works """ client, repo_name = sandbox("wait_commit") repo2_name = util.create_test_repo(client, "wait_commit2") # Create provenance between repos (which creates a new commit) client.create_branch( repo2_name, "master", provenance=[ pfs_proto.Branch(repo=pfs_proto.Repo(name=repo_name, type="user"), name="master") ], ) # Head commit is still open in repo2 client.finish_commit((repo2_name, "master")) with client.commit(repo_name, "master") as c: client.put_file_bytes(c, "input.json", b"hello world") client.finish_commit((repo2_name, "master")) # Just block until all of the commits are yielded commits = client.wait_commit(c.id) assert len(commits) == 2 assert commits[1].finished with client.commit(repo_name, "master") as c2: client.put_file_bytes(c2, "input.json", b"bye world") client.finish_commit((repo2_name, "master")) # Just block until the commit in repo1 is finished commits = client.wait_commit(c2) assert len(commits) == 1 assert commits[0].finished files = list(client.list_file(c2, "/")) assert len(files) == 1
def test_drop_commit(): client, repo_name = sandbox("drop_commit") repo2_name = util.create_test_repo(client, "drop_commit2") # Create provenance between repos (which creates an auto commit) client.create_branch( repo2_name, "master", provenance=[ pfs_proto.Branch(repo=pfs_proto.Repo(name=repo_name, type="user"), name="master") ], ) # Head commit is still open in repo2 client.finish_commit((repo2_name, "master")) with client.commit(repo_name, "master"): pass client.finish_commit((repo2_name, "master")) with client.commit(repo_name, "master") as commit2: pass client.finish_commit((repo2_name, "master")) client.wait_commit(commit2.id) commits = list(client.list_commit(repo_name)) assert len(commits) == 2 client.drop_commit(commit2.id) commits = list(client.list_commit(repo_name)) assert len(commits) == 1 commits = list(client.list_commit(repo2_name)) assert len(commits) == 0 # since list_commit defaults to user commits commits = list( client.list_commit(repo2_name, origin_kind=pfs_proto.OriginKind.AUTO)) assert len(commits) == 2
def sandbox(test_name): client = python_pachyderm.Client() repo_name = util.create_test_repo(client, test_name) return client, repo_name
def test_create_python_pipeline(): client = python_pachyderm.Client() repo_name = util.create_test_repo(client, "create_python_pipeline") pfs_input = python_pachyderm.Input(pfs=python_pachyderm.PFSInput(glob="/", repo=repo_name)) pipeline_name = util.test_repo_name("create_python_pipeline", prefix="pipeline") # create some sample data with client.commit(repo_name, "master") as commit: client.put_file_bytes(commit, 'file.dat', b'DATA') # convenience function for verifying expected files exist def check_all_expected_files(extra_source_files, extra_build_files): list(client.flush_commit([c.commit for c in client.list_commit(pipeline_name)])) check_expected_files(client, "{}_build/source".format(pipeline_name), set([ "/", "/main.py", *extra_source_files, ])) check_expected_files(client, "{}_build/build".format(pipeline_name), set([ "/", "/run.sh", *extra_build_files, ])) check_expected_files(client, "{}/master".format(pipeline_name), set([ "/", "/file.dat", ])) # 1) create a pipeline from a directory with a main.py and requirements.txt with tempfile.TemporaryDirectory(suffix="python_pachyderm") as d: with open(os.path.join(d, "main.py"), "w") as f: f.write(TEST_LIB_SOURCE.format(repo_name)) with open(os.path.join(d, "requirements.txt"), "w") as f: f.write(TEST_REQUIREMENTS_SOURCE) python_pachyderm.create_python_pipeline( client, d, input=pfs_input, pipeline_name=pipeline_name, ) check_all_expected_files( ["/requirements.txt"], ["/leftpad-0.1.2-py3-none-any.whl", "/termcolor-1.1.0-py3-none-any.whl"], ) file = list(client.get_file('{}/master'.format(pipeline_name), 'file.dat')) assert file == [b' DATA'] # 2) update pipeline from a directory without a requirements.txt with tempfile.TemporaryDirectory(suffix="python_pachyderm") as d: with open(os.path.join(d, "main.py"), "w") as f: f.write(TEST_STDLIB_SOURCE.format(repo_name)) python_pachyderm.create_python_pipeline( client, d, input=pfs_input, pipeline_name=pipeline_name, update=True, ) check_all_expected_files([], []) file = list(client.get_file('{}/master'.format(pipeline_name), 'file.dat')) assert file == [b'DATA']