def test_create_pipeline_from_request():
client = python_pachyderm.Client()
repo_name = util.create_test_repo(client, "test_create_pipeline_from_request")
pipeline_name = util.test_repo_name("test_create_pipeline_from_request")
# more or less a copy of the opencv demo's edges pipeline spec
client.create_pipeline_from_request(
pps_proto.CreatePipelineRequest(
pipeline=pps_proto.Pipeline(name=pipeline_name),
description="A pipeline that performs image edge detection by using the OpenCV library.",
input=pps_proto.Input(
pfs=pps_proto.PFSInput(
glob="/*",
repo=repo_name,
),
),
transform=pps_proto.Transform(
cmd=["echo", "hi"],
image="pachyderm/opencv",
),
)
)
assert any(p.pipeline.name == pipeline_name for p in list(client.list_pipeline()))
def test_secrets():
client = python_pachyderm.Client()
secret_name = util.test_repo_name("test-secrets")
client.create_secret(
secret_name,
{
"mykey": "my-value",
},
)
secret = client.inspect_secret(secret_name)
assert secret.secret.name == secret_name
secrets = client.list_secret()
assert len(secrets) == 1
assert secrets[0].secret.name == secret_name
client.delete_secret(secret_name)
with pytest.raises(python_pachyderm.RpcError):
client.inspect_secret(secret_name)
secrets = client.list_secret()
assert len(secrets) == 0
def test_create_python_pipeline():
client = python_pachyderm.Client()
repo_name = util.create_test_repo(client, "create_python_pipeline")
pfs_input = python_pachyderm.Input(pfs=python_pachyderm.PFSInput(glob="/", repo=repo_name))
pipeline_name = util.test_repo_name("create_python_pipeline", prefix="pipeline")
# create some sample data
with client.commit(repo_name, "master") as commit:
client.put_file_bytes(commit, 'file.dat', b'DATA')
# convenience function for verifying expected files exist
def check_all_expected_files(extra_source_files, extra_build_files):
list(client.flush_commit([c.commit for c in client.list_commit(pipeline_name)]))
check_expected_files(client, "{}_build/source".format(pipeline_name), set([
"/",
"/main.py",
*extra_source_files,
]))
check_expected_files(client, "{}_build/build".format(pipeline_name), set([
"/",
"/run.sh",
*extra_build_files,
]))
check_expected_files(client, "{}/master".format(pipeline_name), set([
"/",
"/file.dat",
]))
# 1) create a pipeline from a directory with a main.py and requirements.txt
with tempfile.TemporaryDirectory(suffix="python_pachyderm") as d:
with open(os.path.join(d, "main.py"), "w") as f:
f.write(TEST_LIB_SOURCE.format(repo_name))
with open(os.path.join(d, "requirements.txt"), "w") as f:
f.write(TEST_REQUIREMENTS_SOURCE)
python_pachyderm.create_python_pipeline(
client, d,
input=pfs_input,
pipeline_name=pipeline_name,
)
check_all_expected_files(
["/requirements.txt"],
["/leftpad-0.1.2-py3-none-any.whl", "/termcolor-1.1.0-py3-none-any.whl"],
)
file = list(client.get_file('{}/master'.format(pipeline_name), 'file.dat'))
assert file == [b' DATA']
# 2) update pipeline from a directory without a requirements.txt
with tempfile.TemporaryDirectory(suffix="python_pachyderm") as d:
with open(os.path.join(d, "main.py"), "w") as f:
f.write(TEST_STDLIB_SOURCE.format(repo_name))
python_pachyderm.create_python_pipeline(
client, d,
input=pfs_input,
pipeline_name=pipeline_name,
update=True,
)
check_all_expected_files([], [])
file = list(client.get_file('{}/master'.format(pipeline_name), 'file.dat'))
assert file == [b'DATA']
def create_repo_request():
return transaction_pb2.TransactionRequest(
create_repo=pfs_pb2.CreateRepoRequest(repo=pfs_pb2.Repo(
name=util.test_repo_name("test_batch_transaction"))))
def create_repo_request():
return python_pachyderm.TransactionRequest(
create_repo=python_pachyderm.CreateRepoRequest(
repo=python_pachyderm.Repo(
name=util.test_repo_name("test_batch_transaction"))))