def __init__(self, repo_name, branch: str = 'master'):
self.__client = python_pachyderm.PfsClient(
host=constant.CONFIG['pachd_host'],
port=constant.CONFIG['pachd_port'])
self.__repo_name = repo_name
self.__branch = branch
def __init__(self, test_name):
pfs_client = python_pachyderm.PfsClient()
pps_client = python_pachyderm.PpsClient()
repo_name_suffix = random_string(6)
input_repo_name = "{}-input-{}".format(test_name, repo_name_suffix)
pipeline_repo_name = "{}-pipeline-{}".format(test_name,
repo_name_suffix)
pfs_client.create_repo(input_repo_name,
"input repo for {}".format(test_name))
pps_client.create_pipeline(
pipeline_repo_name,
transform=python_pachyderm.Transform(
cmd=["sh"],
image="alpine",
stdin=["cp /pfs/{}/*.dat /pfs/out/".format(input_repo_name)]),
input=python_pachyderm.Input(pfs=python_pachyderm.PFSInput(
glob="/*", repo=input_repo_name)),
enable_stats=True,
)
with pfs_client.commit(input_repo_name, 'master') as commit:
pfs_client.put_file_bytes(commit, 'file.dat', b'DATA')
self.pps_client = pps_client
self.pfs_client = pfs_client
self.commit = commit
self.input_repo_name = input_repo_name
self.pipeline_repo_name = pipeline_repo_name
def test_client_init_with_env_vars(monkeypatch):
# GIVEN a Pachyderm deployment
# WHEN environment variables are set for Pachyderm host and port
monkeypatch.setenv('PACHD_ADDRESS', 'pachd.example.com:12345')
# AND a client is created without specifying a host or port
client = python_pachyderm.PfsClient()
# THEN the GRPC channel should reflect the host and port specified in the environment variables
assert client.channel._channel.target() == b'pachd.example.com:12345'
def pfs_client():
"""Connect to Pachyderm before tests and reset to initial state after tests."""
# Setup : create a PfsClient instance
client = python_pachyderm.PfsClient()
yield client # this is where the testing happens
# Teardown : Delete all repos, commits, files, pipelines and jobs. This resets the cluster to its initial state.
client.delete_all()
def connect(host, port):
"""
:param host:
:param port:
:return:
"""
client = pachyderm.PfsClient(host, port)
return client
def test_delete_all_repos():
pfs_client = python_pachyderm.PfsClient()
create_repo(pfs_client, "delete_all_1")
create_repo(pfs_client, "delete_all_2")
assert len(pfs_client.list_repo()) >= 2
pfs_client.delete_all_repos()
assert len(pfs_client.list_repo()) == 0
def pfs_client_with_repo(request):
"""Connect to Pachyderm before tests and reset to initial state after tests."""
# Setup : create a PfsClient instance and create a repository
client = python_pachyderm.PfsClient()
client.create_repo(request.param.name, request.param.description)
yield client, request.param # this is where the testing happens
# Teardown : Delete all repos, commits, files, pipelines and jobs. This resets the cluster to its initial state.
client.delete_all()
def __init__(self, config, id):
self.config = config
self.id = id
# Note, auth is only available in enterprise. Currently not supported by the HSC.
# http://docs.pachyderm.io/en/latest/enterprise/auth.html#understanding-pachyderm-access-controls
self.repo_name = config['repo']
self.branch = config['branch']
self.client = python_pachyderm.PfsClient(host=config['host'],
port=config['port'])
self.confirmed_repo_exists = False
def test_pfs_client_init_with_args():
# GIVEN a Pachyderm deployment
# WHEN a client is created with host and port arguments
client = python_pachyderm.PfsClient(host='pachd.example.com', port=54321)
# THEN the GRPC channel should reflect the host and port specified in the arguments
assert client.channel._channel.target() == b'pachd.example.com:54321'
def test_pfs_client_init_with_default_host_port():
# GIVEN a Pachyderm deployment
# WHEN a client is created without specifying a host or port
client = python_pachyderm.PfsClient()
# THEN the GRPC channel should reflect the default of localhost and port 30650
assert client.channel._channel.target() == b'localhost:30650'
def test_delete_non_existent_repo():
pfs_client = python_pachyderm.PfsClient()
orig_repo_count = len(pfs_client.list_repo())
pfs_client.delete_repo('BOGUS_NAME')
assert len(pfs_client.list_repo()) == orig_repo_count
def sandbox(test_name):
client = python_pachyderm.PfsClient()
repo_name = create_repo(client, test_name)
return client, repo_name
from flask import Flask, request, jsonify
import python_pachyderm
import datetime
from grpc import RpcError
# Pachyderm client to fetch and put file in Pachyderm repos
client = python_pachyderm.PfsClient()
app = Flask(__name__)
def gen_filename(prefix):
"""
Generate a filename <prefix>-<current time in ISO8601>.txt
"""
ts = datetime.datetime.now().timestamp()
readable = datetime.datetime.fromtimestamp(ts).isoformat()
return prefix + "-" + readable + ".txt"
@app.route("/train", methods=["POST"])
def add_training_data():
"""
Submit training sample(s) to `training` repo, which will trigger
`train` pipeline.
POST {
"data": [sample_1, sample_2, ..., sample_n]
}
where sample_i: "<user_id> <movie_id> <rate> <timestamp>"
Response: {
