def __init__(self, repo_name, branch: str = 'master'): self.__client = python_pachyderm.PfsClient( host=constant.CONFIG['pachd_host'], port=constant.CONFIG['pachd_port']) self.__repo_name = repo_name self.__branch = branch
def __init__(self, test_name): pfs_client = python_pachyderm.PfsClient() pps_client = python_pachyderm.PpsClient() repo_name_suffix = random_string(6) input_repo_name = "{}-input-{}".format(test_name, repo_name_suffix) pipeline_repo_name = "{}-pipeline-{}".format(test_name, repo_name_suffix) pfs_client.create_repo(input_repo_name, "input repo for {}".format(test_name)) pps_client.create_pipeline( pipeline_repo_name, transform=python_pachyderm.Transform( cmd=["sh"], image="alpine", stdin=["cp /pfs/{}/*.dat /pfs/out/".format(input_repo_name)]), input=python_pachyderm.Input(pfs=python_pachyderm.PFSInput( glob="/*", repo=input_repo_name)), enable_stats=True, ) with pfs_client.commit(input_repo_name, 'master') as commit: pfs_client.put_file_bytes(commit, 'file.dat', b'DATA') self.pps_client = pps_client self.pfs_client = pfs_client self.commit = commit self.input_repo_name = input_repo_name self.pipeline_repo_name = pipeline_repo_name
def test_client_init_with_env_vars(monkeypatch): # GIVEN a Pachyderm deployment # WHEN environment variables are set for Pachyderm host and port monkeypatch.setenv('PACHD_ADDRESS', 'pachd.example.com:12345') # AND a client is created without specifying a host or port client = python_pachyderm.PfsClient() # THEN the GRPC channel should reflect the host and port specified in the environment variables assert client.channel._channel.target() == b'pachd.example.com:12345'
def pfs_client(): """Connect to Pachyderm before tests and reset to initial state after tests.""" # Setup : create a PfsClient instance client = python_pachyderm.PfsClient() yield client # this is where the testing happens # Teardown : Delete all repos, commits, files, pipelines and jobs. This resets the cluster to its initial state. client.delete_all()
def connect(host, port): """ :param host: :param port: :return: """ client = pachyderm.PfsClient(host, port) return client
def test_delete_all_repos(): pfs_client = python_pachyderm.PfsClient() create_repo(pfs_client, "delete_all_1") create_repo(pfs_client, "delete_all_2") assert len(pfs_client.list_repo()) >= 2 pfs_client.delete_all_repos() assert len(pfs_client.list_repo()) == 0
def pfs_client_with_repo(request): """Connect to Pachyderm before tests and reset to initial state after tests.""" # Setup : create a PfsClient instance and create a repository client = python_pachyderm.PfsClient() client.create_repo(request.param.name, request.param.description) yield client, request.param # this is where the testing happens # Teardown : Delete all repos, commits, files, pipelines and jobs. This resets the cluster to its initial state. client.delete_all()
def __init__(self, config, id): self.config = config self.id = id # Note, auth is only available in enterprise. Currently not supported by the HSC. # http://docs.pachyderm.io/en/latest/enterprise/auth.html#understanding-pachyderm-access-controls self.repo_name = config['repo'] self.branch = config['branch'] self.client = python_pachyderm.PfsClient(host=config['host'], port=config['port']) self.confirmed_repo_exists = False
def test_pfs_client_init_with_args(): # GIVEN a Pachyderm deployment # WHEN a client is created with host and port arguments client = python_pachyderm.PfsClient(host='pachd.example.com', port=54321) # THEN the GRPC channel should reflect the host and port specified in the arguments assert client.channel._channel.target() == b'pachd.example.com:54321'
def test_pfs_client_init_with_default_host_port(): # GIVEN a Pachyderm deployment # WHEN a client is created without specifying a host or port client = python_pachyderm.PfsClient() # THEN the GRPC channel should reflect the default of localhost and port 30650 assert client.channel._channel.target() == b'localhost:30650'
def test_delete_non_existent_repo(): pfs_client = python_pachyderm.PfsClient() orig_repo_count = len(pfs_client.list_repo()) pfs_client.delete_repo('BOGUS_NAME') assert len(pfs_client.list_repo()) == orig_repo_count
def sandbox(test_name): client = python_pachyderm.PfsClient() repo_name = create_repo(client, test_name) return client, repo_name
from flask import Flask, request, jsonify import python_pachyderm import datetime from grpc import RpcError # Pachyderm client to fetch and put file in Pachyderm repos client = python_pachyderm.PfsClient() app = Flask(__name__) def gen_filename(prefix): """ Generate a filename <prefix>-<current time in ISO8601>.txt """ ts = datetime.datetime.now().timestamp() readable = datetime.datetime.fromtimestamp(ts).isoformat() return prefix + "-" + readable + ".txt" @app.route("/train", methods=["POST"]) def add_training_data(): """ Submit training sample(s) to `training` repo, which will trigger `train` pipeline. POST { "data": [sample_1, sample_2, ..., sample_n] } where sample_i: "<user_id> <movie_id> <rate> <timestamp>" Response: {