def copy_file(self,
source_commit,
source_path,
dest_commit,
dest_path,
overwrite=None):
"""
Efficiently copies files already in PFS. Note that the destination
repo cannot be an output repo, or the copy operation will (as of
1.9.0) silently fail.
Params:
* source_commit: A tuple, string, or `Commit` object representing the
commit for the source file.
* source_path: A string specifying the path of the source file.
* dest_commit: A tuple, string, or `Commit` object representing the
commit for the destination file.
* dest_path: A string specifying the path of the destination file.
* overwrite: Am optional bool specifying whether to overwrite the
destination file if it already exists.
"""
req = proto.CopyFileRequest(
src=proto.File(commit=commit_from(source_commit),
path=source_path),
dst=proto.File(commit=commit_from(dest_commit), path=dest_path),
overwrite=overwrite,
)
self.stub.CopyFile(req, metadata=self.metadata)
def get_file(self,
commit,
path,
offset_bytes=0,
size_bytes=0,
extract_value=True):
"""
Returns an iterator of the contents contents of a file at a specific Commit.
Params:
* commit: A tuple, string, or Commit object representing the commit.
* path: The path of the file.
* offset_bytes: Optional. specifies a number of bytes that should be
skipped in the beginning of the file.
* size_bytes: Optional. limits the total amount of data returned, note
you will get fewer bytes than size if you pass a value larger than the
size of the file. If size is set to 0 then all of the data will be
returned.
* extract_value: If True, then an ExtractValueIterator will be return,
which will iterate over the bytes of the file. If False, then the
protobuf response iterator will return.
"""
req = proto.GetFileRequest(file=proto.File(commit=commit_from(commit),
path=path),
offset_bytes=offset_bytes,
size_bytes=size_bytes)
res = self.stub.GetFile(req, metadata=self.metadata)
if extract_value:
return ExtractValueIterator(res)
return res
def list_file(self, commit, path, history=None, include_contents=None):
"""
Lists the files in a directory.
Params:
* commit: A tuple, string, or `Commit` object representing the commit.
* path: The path to the directory.
* history: An optional int that indicates to return jobs from
historical versions of pipelines. Semantics are:
0: Return jobs from the current version of the pipeline or pipelines.
1: Return the above and jobs from the next most recent version
2: etc.
-1: Return jobs from all historical versions.
* include_contents: An optional bool. If `True`, file contents are
included.
"""
req = proto.ListFileRequest(
file=proto.File(commit=commit_from(commit), path=path),
history=history,
full=include_contents,
)
return self.stub.ListFileStream(req, metadata=self.metadata)
def inspect_file(self, commit, path):
"""
Inspects a file. Returns a `FileInfo` object.
Params:
* commit: A tuple, string, or `Commit` object representing the commit.
* path: A string specifying the path to the file.
"""
req = proto.InspectFileRequest(
file=proto.File(commit=commit_from(commit), path=path))
return self.stub.InspectFile(req, metadata=self.metadata)
def inspect_file(self, commit, path):
"""
Returns info about a specific file.
Params:
* commit: A tuple, string, or Commit object representing the commit.
* path: Path to file.
"""
req = proto.InspectFileRequest(file=proto.File(commit=commit_from(commit), path=path))
res = self.stub.InspectFile(req, metadata=self.metadata)
return res
def walk_file(self, commit, path):
"""
Walks over all descendant files in a directory. Returns a generator of
`FileInfo` objects.
Params:
* commit: A tuple, string, or `Commit` object representing the commit.
* path: The path to the directory.
"""
commit = commit_from(commit)
f = proto.File(commit=commit_from(commit), path=path)
req = proto.WalkFileRequest(file=f)
return self.stub.WalkFile(req, metadata=self.metadata)
def wrap(value):
yield proto.PutFileRequest(
file=proto.File(commit=commit_from(commit), path=path),
value=value[:BUFFER_SIZE],
delimiter=delimiter,
target_file_datums=target_file_datums,
target_file_bytes=target_file_bytes,
overwrite_index=overwrite_index_proto)
for i in range(BUFFER_SIZE, len(value), BUFFER_SIZE):
yield proto.PutFileRequest(
value=value[i:i + BUFFER_SIZE],
overwrite_index=overwrite_index_proto)
def wrap(value):
for i, chunk in enumerate(value):
if i == 0:
yield proto.PutFileRequest(
file=proto.File(commit=commit_from(commit),
path=path),
value=chunk,
delimiter=delimiter,
target_file_datums=target_file_datums,
target_file_bytes=target_file_bytes,
overwrite_index=overwrite_index_proto)
else:
yield proto.PutFileRequest(value=chunk)
def delete_file(self, commit, path):
"""
Deletes a file from a Commit. DeleteFile leaves a tombstone in the
Commit, assuming the file isn't written to later attempting to get the
file from the finished commit will result in not found error. The file
will of course remain intact in the Commit's parent.
Params:
* commit: A tuple, string, or `Commit` object representing the commit.
* path: The path to the file.
"""
req = proto.DeleteFileRequest(
file=proto.File(commit=commit_from(commit), path=path))
self.stub.DeleteFile(req, metadata=self.metadata)
def wrap(value):
for i in itertools.count():
chunk = value.read(BUFFER_SIZE)
if len(chunk) == 0:
return
if i == 0:
yield proto.PutFileRequest(
file=proto.File(commit=commit_from(commit),
path=path),
value=chunk,
delimiter=delimiter,
target_file_datums=target_file_datums,
target_file_bytes=target_file_bytes,
overwrite_index=overwrite_index_proto)
else:
yield proto.PutFileRequest(value=chunk)
def put_file_url(self, commit, path, url, recursive=False):
"""
Puts a file using the content found at a URL. The URL is sent to the
server which performs the request.
Params:
* commit: A tuple, string, or Commit object representing the commit.
* path: The path to the file.
* url: The url of the file to put.
* recursive: allow for recursive scraping of some types URLs for
example on s3:// urls.
"""
req = iter([
proto.PutFileRequest(file=proto.File(commit=commit_from(commit),
path=path),
url=url,
recursive=recursive)
])
self.stub.PutFile(req, metadata=self.metadata)
def put_file_url(self, commit, path, url, recursive=None):
"""
Puts a file using the content found at a URL. The URL is sent to the
server which performs the request. Note that this is not a standard
PFS function.
Params:
* commit: A tuple, string, or `Commit` object representing the commit.
* path: A string specifying the path to the file.
* url: A string specifying the url of the file to put.
* recursive: allow for recursive scraping of some types URLs, for
example on s3:// URLs.
"""
req = iter([
proto.PutFileRequest(file=proto.File(commit=commit_from(commit),
path=path),
url=url,
recursive=recursive)
])
self.stub.PutFile(req, metadata=self.metadata)
def get_file(self, commit, path, offset_bytes=None, size_bytes=None):
"""
Returns an iterator of the contents of a file at a specific commit.
Params:
* commit: A tuple, string, or `Commit` object representing the commit.
* path: A string specifying the path of the file.
* offset_bytes: An optional int. Specifies a number of bytes that
should be skipped in the beginning of the file.
* size_bytes: An optional int. limits the total amount of data
returned, note you will get fewer bytes than size if you pass a value
larger than the size of the file. If size is set to 0 then all of the
data will be returned.
"""
req = proto.GetFileRequest(file=proto.File(commit=commit_from(commit),
path=path),
offset_bytes=offset_bytes,
size_bytes=size_bytes)
res = self.stub.GetFile(req, metadata=self.metadata)
for item in res:
yield item.value
def list_file(self, commit, path, recursive=False):
"""
Lists the files in a directory.
Params:
* commit: A tuple, string, or Commit object representing the commit.
* path: The path to the directory.
* recursive: If True, continue listing the files for sub-directories.
"""
req = proto.ListFileRequest(
file=proto.File(commit=commit_from(commit), path=path))
res = self.stub.ListFile(req, metadata=self.metadata)
file_infos = res.file_info
if recursive:
dirs = [f for f in file_infos if f.file_type == proto.DIR]
files = [f for f in file_infos if f.file_type == proto.FILE]
return sum(
[self.list_file(commit, d.file.path, recursive) for d in dirs],
files)
return list(file_infos)