def _req(self,
grpc_service: Service,
grpc_method_name: str,
req=None,
**kwargs):
# For proto request objects with no equivalent betterproto
if "pb2" in type(req).__module__ or grpc_method_name == "ModifyFile":
return _Client._req(self, grpc_service, grpc_method_name, req,
**kwargs)
# convert betterproto to google protobuf
if req is not None:
req = bp_to_pb(req)
else:
for k, v in kwargs.items():
if (isinstance(v, list) and hasattr(v[0], "__module__")
and "proto" in v[0].__module__):
kwargs[k] = [bp_to_pb(bp) for bp in kwargs[k]]
elif hasattr(v, "__module__") and "proto" in v.__module__:
kwargs[k] = bp_to_pb(v)
# gRPC call
pb = _Client._req(self, grpc_service, grpc_method_name, req, **kwargs)
return pb_to_bp(pb)
def create_test_pipeline(client: python_pachyderm.Client, test_name):
repo_name_suffix = random_string(6)
input_repo_name = create_test_repo(client,
test_name,
prefix="input",
suffix=repo_name_suffix)
pipeline_repo_name = test_repo_name(test_name,
prefix="pipeline",
suffix=repo_name_suffix)
client.create_pipeline(
pipeline_repo_name,
transform=pps_proto.Transform(
cmd=["sh"],
image="alpine",
stdin=["cp /pfs/{}/*.dat /pfs/out/".format(input_repo_name)],
),
input=pps_proto.Input(
pfs=pps_proto.PfsInput(glob="/*", repo=input_repo_name)),
)
# TODO figre out what is actually happening here
with client.commit(input_repo_name, "master") as commit:
client.put_file_bytes(commit, "file.dat", b"DATA")
return (commit, input_repo_name, pipeline_repo_name)
def main():
print("connecting to pachd")
client = Client()
print("connected")
while True:
# Polls queue
msgs = receive_message()
if msgs:
with client.commit("spout", "master") as c:
for msg in msgs:
# hash the file to assign unique name
filename = hashlib.sha256(msg).hexdigest() + ".txt"
client.put_file_bytes(c, filename, msg)
def check_expected_files(client: python_pachyderm.Client, commit, expected):
for fi in client.walk_file(commit, "/"):
path = fi.file.path
assert path in expected, "unexpected path: {}".format(path)
expected.remove(path)
for path in expected:
assert False, "expected path not found: {}".format(path)
def __init__(
self,
host: str = None,
port: int = None,
auth_token: str = None,
root_certs: bytes = None,
transaction_id: str = None,
tls: bool = None,
use_default_host: bool = True,
):
_Client.__init__(
self,
host,
port,
auth_token,
root_certs,
transaction_id,
tls,
use_default_host,
)
def get_latest_job(client: Client, pipeline_name: str) -> JobInfo:
max_milliseconds = 0
latest_job = None
for job_info in client.list_job(pipeline_name=pipeline_name,
history=0,
full=False):
state = job_info.state
if state == 3: # 3 means the job is complete
started_milliseconds = job_info.started.seconds
if started_milliseconds > max_milliseconds:
max_milliseconds = started_milliseconds
latest_job = job_info
return latest_job
def polling_consumer():
while True:
# Polls queue
msgs = receive_message()
if msgs:
print("connecting to pachd")
client = Client.new_from_config()
print("connected")
with client.commit('spout', 'master') as c:
for msg in msgs:
# hash the file to assign unique name
filename = hashlib.sha256(msg).hexdigest() + ".txt"
client.put_file_bytes(c, filename, msg)
def put_files(client: Client, source_path: str, commit: SubcommitType,
dest_path: str, **kwargs) -> None:
"""Utility function for inserting files from the local `source_path`
into Pachyderm. Roughly equivalent to ``pachctl put file [-r]``.
Parameters
----------
client : Client
A python_pachyderm client instance.
source_path : str
The file/directory to recursively insert content from.
commit : SubcommitType
The open commit to add files to.
dest_path : str
The destination path in PFS.
**kwargs : dict
Keyword arguments to forward. See
``ModifyFileClient.put_file_from_filepath()`` for more details.
Examples
--------
>>> source_dir = "data/training/
>>> with client.commit("repo_name", "master") as commit:
>>> python_pachyderm.put_files(client, source_dir, commit, "/training_set/")
...
>>> with client.commit("repo_name", "master") as commit2:
>>> python_pachyderm.put_files(client, "metadata/params.csv", commit2, "/hyperparams.csv")
>>> python_pachyderm.put_files(client, "spec.json", commit2, "/spec.json")
.. # noqa: W505
"""
with client.modify_file_client(commit) as mfc:
if os.path.isfile(source_path):
mfc.put_file_from_filepath(dest_path, source_path, **kwargs)
elif os.path.isdir(source_path):
for root, _, filenames in os.walk(source_path):
for filename in filenames:
source_filepath = os.path.join(root, filename)
dest_filepath = os.path.join(
dest_path,
os.path.relpath(source_filepath, start=source_path))
mfc.put_file_from_filepath(dest_filepath, source_filepath,
**kwargs)
else:
raise Exception("Please provide an existing directory or file")