def commit_from(src, allow_just_repo=False):
if isinstance(src, pfs_proto.Commit):
return src
elif isinstance(src, (tuple, list)) and len(src) == 2:
return pfs_proto.Commit(repo=pfs_proto.Repo(name=src[0]), id=src[1])
elif isinstance(src, six.string_types):
repo_name, commit_id = src.split('/', 1)
return pfs_proto.Commit(repo=pfs_proto.Repo(name=repo_name),
id=commit_id)
if not allow_just_repo:
raise ValueError("Invalid commit type")
return pfs_proto.Commit(repo=pfs_proto.Repo(name=src))
def start_commit(self,
repo_name,
branch=None,
parent=None,
description=None,
provenance=None):
"""
Begins the process of committing data to a Repo. Once started you can
write to the Commit with PutFile and when all the data has been
written you must finish the Commit with FinishCommit. NOTE, data is
not persisted until FinishCommit is called. A Commit object is
returned.
Params:
* repo_name: A string specifying the name of the repo.
* branch: A string specifying the branch name. This is a more
convenient way to build linear chains of commits. When a commit is
started with a non-empty branch the value of branch becomes an alias
for the created Commit. This enables a more intuitive access pattern.
When the commit is started on a branch the previous head of the branch
is used as the parent of the commit.
* parent: An optional `Commit` object specifying the parent commit.
Upon creation the new commit will appear identical to the parent
commit, data can safely be added to the new commit without affecting
the contents of the parent commit.
* description: An optional string describing the commit.
* provenance: An optional iterable of `CommitProvenance` objects
specifying the commit provenance.
"""
req = proto.StartCommitRequest(
parent=proto.Commit(repo=proto.Repo(name=repo_name), id=parent),
branch=branch,
description=description,
provenance=provenance,
)
return self.stub.StartCommit(req, metadata=self.metadata)
def start_commit(self,
repo_name,
branch=None,
parent=None,
description=None):
"""
Begins the process of committing data to a Repo. Once started you can
write to the Commit with PutFile and when all the data has been
written you must finish the Commit with FinishCommit. NOTE, data is
not persisted until FinishCommit is called. A Commit object is
returned.
Params:
* repo_name: The name of the repo.
* branch: A more convenient way to build linear chains of commits.
When a commit is started with a non-empty branch the value of branch
becomes an alias for the created Commit. This enables a more intuitive
access pattern. When the commit is started on a branch the previous
head of the branch is used as the parent of the commit.
* parent: Specifies the parent Commit, upon creation the new Commit
will appear identical to the parent Commit, data can safely be added
to the new commit without affecting the contents of the parent Commit.
You may pass "" as parentCommit in which case the new Commit will have
no parent and will initially appear empty.
* description: (optional) explanation of the commit for clarity.
"""
req = proto.StartCommitRequest(parent=proto.Commit(
repo=proto.Repo(name=repo_name), id=parent),
branch=branch,
description=description)
res = self.stub.StartCommit(req, metadata=self.metadata)
return res
def inspect_branch(self, repo_name, branch_name):
"""
Inspects a branch. Returns a `BranchInfo` object.
"""
branch = proto.Branch(repo=proto.Repo(name=repo_name), name=branch_name)
req = proto.InspectBranchRequest(branch=branch)
return self.stub.InspectBranch(req, metadata=self.metadata)
def flush_commit(self, commits, repos=None):
"""
Blocks until all of the commits which have a set of commits as
provenance have finished. For commits to be considered they must have
all of the specified commits as provenance. This in effect waits for
all of the jobs that are triggered by a set of commits to complete.
It returns an error if any of the commits it's waiting on are
cancelled due to one of the jobs encountering an error during runtime.
Note that it's never necessary to call FlushCommit to run jobs,
they'll run no matter what, FlushCommit just allows you to wait for
them to complete and see their output once they do. This returns an
iterator of CommitInfo objects.
Yields `CommitInfo` objects.
Params:
* commits: A list of tuples, strings, or `Commit` objects representing
the commits to flush.
* repos: An optional list of strings specifying repo names. If
specified, only commits within these repos will be flushed.
"""
to_repos = [proto.Repo(name=r) for r in repos] if repos is not None else None
req = proto.FlushCommitRequest(commits=[commit_from(c) for c in commits],
to_repos=to_repos)
return self.stub.FlushCommit(req, metadata=self.metadata)
def list_commit(self,
repo_name,
to_commit=None,
from_commit=None,
number=0):
"""
Gets a list of CommitInfo objects.
Params:
* repo_name: If only `repo_name` is given, all commits in the repo are
returned.
* to_commit: Optional. Only the ancestors of `to`, including `to`
itself, are considered.
* from_commit: Optional. Only the descendants of `from`, including
`from` itself, are considered.
* number: Optional. Determines how many commits are returned. If
`number` is 0, all commits that match the aforementioned criteria are
returned.
"""
req = proto.ListCommitRequest(repo=proto.Repo(name=repo_name),
number=number)
if to_commit is not None:
req.to.CopyFrom(commit_from(to_commit))
if from_commit is not None:
getattr(req, 'from').CopyFrom(commit_from(from_commit))
res = self.stub.ListCommit(req, metadata=self.metadata)
if hasattr(res, 'commit_info'):
return res.commit_info
return []
def flush_commit(self, commits, repos=tuple()):
"""
Blocks until all of the commits which have a set of commits as
provenance have finished. For commits to be considered they must have
all of the specified commits as provenance. This in effect waits for
all of the jobs that are triggered by a set of commits to complete.
It returns an error if any of the commits it's waiting on are
cancelled due to one of the jobs encountering an error during runtime.
Note that it's never necessary to call FlushCommit to run jobs,
they'll run no matter what, FlushCommit just allows you to wait for
them to complete and see their output once they do. This returns an
iterator of CommitInfo objects.
Params:
* commits: A commit or a list of commits to wait on.
* repos: Optional. Only the commits up to and including those repos.
will be considered, otherwise all repos are considered.
"""
req = proto.FlushCommitRequest(
commits=[commit_from(c) for c in commits],
to_repos=[proto.Repo(name=r) for r in repos])
res = self.stub.FlushCommit(req, metadata=self.metadata)
for commit in res:
yield commit
def subscribe_commit(self,
repo_name,
branch,
from_commit_id=None,
state=None):
"""
Yields `CommitInfo` objects as commits occur.
Params:
* repo_name: A string specifying the name of the repo.
* branch: A string specifying branch to subscribe to.
* from_commit_id: An optional string specifying the commit ID. Only
commits created since this commit are returned.
* state: The commit state to filter on.
"""
repo = proto.Repo(name=repo_name)
req = proto.SubscribeCommitRequest(repo=repo,
branch=branch,
state=state)
if from_commit_id is not None:
getattr(req,
'from').CopyFrom(proto.Commit(repo=repo,
id=from_commit_id))
return self.stub.SubscribeCommit(req, metadata=self.metadata)
def inspect_repo(self, repo_name):
"""
Returns info about a specific repo. Returns a `RepoInfo` object.
Params:
* repo_name: Name of the repo.
"""
req = proto.InspectRepoRequest(repo=proto.Repo(name=repo_name))
return self.stub.InspectRepo(req, metadata=self.metadata)
def list_branch(self, repo_name):
"""
Lists the active branch objects on a repo. Returns a list of
`BranchInfo` objects.
Params:
* repo_name: A string specifying the repo name.
"""
req = proto.ListBranchRequest(repo=proto.Repo(name=repo_name))
res = self.stub.ListBranch(req, metadata=self.metadata)
return res.branch_info
def delete_repo(self, repo_name, force=None):
"""
Deletes a repo and reclaims the storage space it was using.
Params:
* repo_name: The name of the repo.
* force: If set to true, the repo will be removed regardless of
errors. This argument should be used with care.
"""
req = proto.DeleteRepoRequest(repo=proto.Repo(name=repo_name), force=force, all=False)
self.stub.DeleteRepo(req, metadata=self.metadata)
def list_branch(self, repo_name):
"""
Lists the active Branch objects on a Repo.
Params:
* repo_name: The name of the repo.
"""
req = proto.ListBranchRequest(repo=proto.Repo(name=repo_name))
res = self.stub.ListBranch(req, metadata=self.metadata)
if hasattr(res, 'branch_info'):
return res.branch_info
return []
def create_repo(self, repo_name, description=None):
"""
Creates a new Repo object in PFS with the given name. Repos are the
top level data object in PFS and should be used to store data of a
similar type. For example rather than having a single Repo for an
entire project you might have separate Repos for logs, metrics,
database dumps etc.
Params:
* repo_name: Name of the repo.
* description: Repo description.
"""
req = proto.CreateRepoRequest(repo=proto.Repo(name=repo_name), description=description)
self.stub.CreateRepo(req, metadata=self.metadata)
def delete_branch(self, repo_name, branch_name, force=None):
"""
Deletes a branch, but leaves the commits themselves intact. In other
words, those commits can still be accessed via commit IDs and other
branches they happen to be on.
Params:
* repo_name: A string specifying the repo name.
* branch_name: A string specifying the name of the branch to delete.
* force: A bool specifying whether to force the branch deletion.
"""
branch = proto.Branch(repo=proto.Repo(name=repo_name),
name=branch_name)
req = proto.DeleteBranchRequest(branch=branch, force=force)
self.stub.DeleteBranch(req, metadata=self.metadata)
def create_repo(self, repo_name, description=None, update=None):
"""
Creates a new `Repo` object in PFS with the given name. Repos are the
top level data object in PFS and should be used to store data of a
similar type. For example rather than having a single `Repo` for an
entire project you might have separate `Repo`s for logs, metrics,
database dumps etc.
Params:
* repo_name: Name of the repo.
* description: An optional string describing the repo.
* update: Whether to update if the repo already exists.
"""
req = proto.CreateRepoRequest(repo=proto.Repo(name=repo_name),
description=description,
update=update)
self.stub.CreateRepo(req, metadata=self.metadata)
def subscribe_commit(self, repo_name, branch, from_commit_id=None):
"""
SubscribeCommit is like ListCommit but it keeps listening for commits as
they come in. This returns an iterator Commit objects.
Params:
* repo_name: Name of the repo.
* branch: Branch to subscribe to.
* from_commit_id: Optional. Only commits created since this commit
are returned.
"""
repo = proto.Repo(name=repo_name)
req = proto.SubscribeCommitRequest(repo=repo, branch=branch)
if from_commit_id is not None:
getattr(req, 'from').CopyFrom(proto.Commit(repo=repo, id=from_commit_id))
res = self.stub.SubscribeCommit(req, metadata=self.metadata)
return res
def create_branch(self, repo_name, branch_name, commit=None, provenance=None):
"""
Creates a new branch.
Params:
* repo_name: A string specifying the name of the repo.
* branch_name: A string specifying the new branch name.
* commit: An optional tuple, string, or `Commit` object representing
the head commit of the branch.
* provenance: An optional iterable of `Branch` objects representing
the branch provenance.
"""
req = proto.CreateBranchRequest(
branch=proto.Branch(repo=proto.Repo(name=repo_name), name=branch_name),
head=commit_from(commit) if commit is not None else None,
provenance=provenance,
)
self.stub.CreateBranch(req, metadata=self.metadata)
def delete_repo(self, repo_name=None, force=False, all=False):
"""
Deletes a repo and reclaims the storage space it was using.
Params:
* repo_name: The name of the repo.
* force: If set to true, the repo will be removed regardless of
errors. This argument should be used with care.
* all: Delete all repos.
"""
if not all:
if repo_name:
req = proto.DeleteRepoRequest(repo=proto.Repo(name=repo_name), force=force)
self.stub.DeleteRepo(req, metadata=self.metadata)
else:
raise ValueError("Either a repo_name or all=True needs to be provided")
else:
if not repo_name:
req = proto.DeleteRepoRequest(force=force, all=all)
self.stub.DeleteRepo(req, metadata=self.metadata)
else:
raise ValueError("Cannot specify a repo_name if all=True")
