def _req(self, grpc_service: Service, grpc_method_name: str, req=None, **kwargs): # For proto request objects with no equivalent betterproto if "pb2" in type(req).__module__ or grpc_method_name == "ModifyFile": return _Client._req(self, grpc_service, grpc_method_name, req, **kwargs) # convert betterproto to google protobuf if req is not None: req = bp_to_pb(req) else: for k, v in kwargs.items(): if (isinstance(v, list) and hasattr(v[0], "__module__") and "proto" in v[0].__module__): kwargs[k] = [bp_to_pb(bp) for bp in kwargs[k]] elif hasattr(v, "__module__") and "proto" in v.__module__: kwargs[k] = bp_to_pb(v) # gRPC call pb = _Client._req(self, grpc_service, grpc_method_name, req, **kwargs) return pb_to_bp(pb)
def create_test_pipeline(client: python_pachyderm.Client, test_name): repo_name_suffix = random_string(6) input_repo_name = create_test_repo(client, test_name, prefix="input", suffix=repo_name_suffix) pipeline_repo_name = test_repo_name(test_name, prefix="pipeline", suffix=repo_name_suffix) client.create_pipeline( pipeline_repo_name, transform=pps_proto.Transform( cmd=["sh"], image="alpine", stdin=["cp /pfs/{}/*.dat /pfs/out/".format(input_repo_name)], ), input=pps_proto.Input( pfs=pps_proto.PfsInput(glob="/*", repo=input_repo_name)), ) # TODO figre out what is actually happening here with client.commit(input_repo_name, "master") as commit: client.put_file_bytes(commit, "file.dat", b"DATA") return (commit, input_repo_name, pipeline_repo_name)
def main(): print("connecting to pachd") client = Client() print("connected") while True: # Polls queue msgs = receive_message() if msgs: with client.commit("spout", "master") as c: for msg in msgs: # hash the file to assign unique name filename = hashlib.sha256(msg).hexdigest() + ".txt" client.put_file_bytes(c, filename, msg)
def check_expected_files(client: python_pachyderm.Client, commit, expected): for fi in client.walk_file(commit, "/"): path = fi.file.path assert path in expected, "unexpected path: {}".format(path) expected.remove(path) for path in expected: assert False, "expected path not found: {}".format(path)
def __init__( self, host: str = None, port: int = None, auth_token: str = None, root_certs: bytes = None, transaction_id: str = None, tls: bool = None, use_default_host: bool = True, ): _Client.__init__( self, host, port, auth_token, root_certs, transaction_id, tls, use_default_host, )
def get_latest_job(client: Client, pipeline_name: str) -> JobInfo: max_milliseconds = 0 latest_job = None for job_info in client.list_job(pipeline_name=pipeline_name, history=0, full=False): state = job_info.state if state == 3: # 3 means the job is complete started_milliseconds = job_info.started.seconds if started_milliseconds > max_milliseconds: max_milliseconds = started_milliseconds latest_job = job_info return latest_job
def polling_consumer(): while True: # Polls queue msgs = receive_message() if msgs: print("connecting to pachd") client = Client.new_from_config() print("connected") with client.commit('spout', 'master') as c: for msg in msgs: # hash the file to assign unique name filename = hashlib.sha256(msg).hexdigest() + ".txt" client.put_file_bytes(c, filename, msg)
def put_files(client: Client, source_path: str, commit: SubcommitType, dest_path: str, **kwargs) -> None: """Utility function for inserting files from the local `source_path` into Pachyderm. Roughly equivalent to ``pachctl put file [-r]``. Parameters ---------- client : Client A python_pachyderm client instance. source_path : str The file/directory to recursively insert content from. commit : SubcommitType The open commit to add files to. dest_path : str The destination path in PFS. **kwargs : dict Keyword arguments to forward. See ``ModifyFileClient.put_file_from_filepath()`` for more details. Examples -------- >>> source_dir = "data/training/ >>> with client.commit("repo_name", "master") as commit: >>> python_pachyderm.put_files(client, source_dir, commit, "/training_set/") ... >>> with client.commit("repo_name", "master") as commit2: >>> python_pachyderm.put_files(client, "metadata/params.csv", commit2, "/hyperparams.csv") >>> python_pachyderm.put_files(client, "spec.json", commit2, "/spec.json") .. # noqa: W505 """ with client.modify_file_client(commit) as mfc: if os.path.isfile(source_path): mfc.put_file_from_filepath(dest_path, source_path, **kwargs) elif os.path.isdir(source_path): for root, _, filenames in os.walk(source_path): for filename in filenames: source_filepath = os.path.join(root, filename) dest_filepath = os.path.join( dest_path, os.path.relpath(source_filepath, start=source_path)) mfc.put_file_from_filepath(dest_filepath, source_filepath, **kwargs) else: raise Exception("Please provide an existing directory or file")