async def clone_split(self, backing_vol: localvolume,
feedback_ui: AbstractFeedbackUI):
backing_vol_repo_path = self._root.GetPathForBackingVolume(backing_vol)
local_worktree_path = self.get_local_repo_path()
remote_url = self.get_remote_url()
if RepoExists(backing_vol_repo_path):
await feedback_ui.error(
"Repository already exists. Cannot clone over it. Maybe just checkout a worktree instead?"
)
return
elif RepoExists(local_worktree_path):
await feedback_ui.error(
"Worktree already exists. Cannot checkout over it.")
return
else:
# clone to backing vol with no worktree
await feedback_ui.output(
"Initializing bare repository on backing volume: " +
backing_vol_repo_path)
git.Git(backing_vol_repo_path).clone("--bare", remote_url,
backing_vol_repo_path)
# add the worktree
backing_repo = git.Repo(backing_vol_repo_path)
await feedback_ui.output("Adding worktree: " + local_worktree_path)
backing_repo.git.worktree("add", local_worktree_path)
async def push_lfs_objects_subroutine(self,
feedback_ui: AbstractFeedbackUI,
branch: str):
"""Push lfs objects to the remote.
lfs is smart about only pushing objects that need to be pushed.
Failure will throw. Therefore, a succesful run of this subroutine guarontees
lfs objects exist."""
server = self.get_server_routines().get_server()
remote_url = self.get_remote_url()
local_path = self.get_local_repo_path()
await feedback_ui.output("Pushing LFS objects: " + local_path +
" to: " + remote_url)
if not RepoExists(local_path):
await feedback_ui.error(
"No local worktree. You might need to create or pull it first."
)
return False
repo = git.Repo(local_path)
remote = create_update_remote(repo, server.get_name(), remote_url)
assert isinstance(remote, git.Remote)
push_output = repo.git.lfs("push", server.get_name(), branch)
await feedback_ui.output(push_output)
await feedback_ui.output("Done pushing LFS objects.")
def is_conflicted(self, group_name: str) -> bool:
server = self.get_server_routines().get_server()
local_path = self.get_server_routines().local_path_for_type_registry(
server.get_name(), group_name, self.get_typename())
if RepoExists(local_path):
repo = git.Repo(local_path)
return is_in_conflict(repo)
async def fetch_master_subroutine(self, group_name: str):
"""Fetches any remote changes but doesn't do anything with them.
"""
server = self.get_server_routines().get_server()
local_path = self.get_server_routines().local_path_for_type_registry(
server.get_name(), group_name, self.get_typename())
server_url = self.get_server_routines(
).remote_path_for_entity_registry(group_name=group_name,
type_name=self.get_typename())
if not RepoExists(local_path):
raise RuntimeError("No local worktree.")
repo = git.Repo(local_path)
master_branch = None
# check for branches and create local if needed.
for branch in repo.heads:
assert isinstance(branch, git.Head)
if branch.name == "master":
master_branch = branch
if master_branch is None:
raise RuntimeError("There is no master branch.")
remote = create_update_remote(repo, server.get_name(), server_url)
remote.fetch("master")
async def init_local(self, group_name: str):
server = self.get_server_routines().get_server()
local_path = self.get_server_routines().local_path_for_type_registry(
server.get_name(), group_name, self.get_typename())
if RepoExists(local_path):
await self.get_feedback_ui().error(
"Repo already exists. Cannot init.")
return
os.makedirs(local_path, exist_ok=True)
git.Repo.init(local_path)
async def clone(self, feedback_ui: AbstractFeedbackUI):
local_repo_path = self.get_local_repo_path()
remote_url = self.get_remote_url()
if RepoExists(local_repo_path):
await feedback_ui.error(
"Repository already exists. Cannot clone over it.")
return
else:
await feedback_ui.output("Cloning from: " + remote_url + " -> " +
local_repo_path)
git.Repo.clone_from(remote_url, local_repo_path)
async def print_status_routine(self, feedback_ui: AbstractFeedbackUI):
existsText = "absent"
statusText = "---"
if RepoExists(self.get_local_repo_path()):
existsText = "exists"
rep = self.get_local_repo()
if rep.is_dirty(True, True, True, False):
statusText = "dirty"
else:
statusText = "clean"
await feedback_ui.output("root: {0} - {1} - {2} - {3}".format(
self.get_local_repo_path(), self.root.GetID(), existsText,
statusText))
async def remote_exists(self, group_name: str):
server = self.get_server_routines().get_server()
local_path = self.get_server_routines().local_path_for_type_registry(
server.get_name(), group_name, self.get_typename())
if not RepoExists(local_path):
await self.get_feedback_ui().error(
"No local worktree. You can create an empty one with init_local or use a pull command to get an existing one."
)
return
repo = git.Repo(local_path)
return exists(repo, server.get_name())
async def import_from_master_subroutine(self,
feedback_ui: AbstractFeedbackUI,
group_name: str):
"""This overwrites the db with the master branch. Therefore, we delete and replace local with master as well.
This helps track and merge changes.
"""
server = self.get_server_routines().get_server()
local_path = self.get_server_routines().local_path_for_type_registry(
server.get_name(), group_name, self.get_typename())
if not RepoExists(local_path):
raise RuntimeError("No local worktree.")
repo = git.Repo(local_path)
master_branch = None
#check for branches and create local if needed.
for branch in repo.heads:
assert isinstance(branch, git.Head)
if branch.name == "master":
master_branch = branch
if master_branch is None:
raise RuntimeError("There is no master branch.")
#check out master
master_branch.checkout(force=True)
if repo.is_dirty():
raise RuntimeError(
"Master branch is dirty after checkout. Won't import dirty data to db."
)
manager_routines = self.get_local_manager_routines()
#clear out the manager before importing
for item in manager_routines.GetAllItems():
name = manager_routines.ItemToName(item)
await manager_routines.DeleteRoutine(name)
await manager_routines.ImportAllRoutine(local_path)
#if we got here, we succesfully imported from master.
#now we kill the local branch completely and create it from master
for branch in repo.heads:
assert isinstance(branch, git.Head)
if branch.name == "local":
git.Head.delete(repo, "local", force=True)
branch_output = repo.git.branch("local", "master")
await feedback_ui.output(branch_output)
async def checkout_routine(self, feedback_ui: AbstractFeedbackUI,
group_name: str):
"""Does a fresh checkout of the master branch from remote. Blows away existing repo and db items."""
server = self.get_server_routines().get_server()
server_url = self.get_server_routines(
).remote_path_for_entity_registry(group_name=group_name,
type_name=self.get_typename())
local_path = self.get_server_routines().local_path_for_type_registry(
server.get_name(), group_name, self.get_typename())
if RepoExists(local_path):
#blow it away
DeleteLocalRepo(local_path)
await self.get_feedback_ui().output("Cloning from: " + server_url)
git.Repo.clone_from(server_url, local_path)
await self.import_from_master_subroutine(feedback_ui, group_name)
async def init_new(self, feedback_ui: AbstractFeedbackUI):
dir = self._root_config.GetWorkingPath(self._mapper)
if RepoExists(dir):
await feedback_ui.error("Already exists.")
return
await feedback_ui.output("Initializing Repo.")
repo = InitWorkingTreeRoot(dir)
await feedback_ui.output("Installing LFS to Repo.")
InstallLFSRepo(repo)
await feedback_ui.output("Setting up .gitignore")
CheckCreateIgnore(repo)
await feedback_ui.output("Commiting to head")
repo.git.commit(m="Initial commit.")
os.chdir(dir)
return
async def delete_local_interactive_routine(
self, group_name: str,
dirty_ui: AbstractModalTrueFalseDefaultQuestionUI):
server = self.get_server_routines().get_server()
local_path = self.get_server_routines().local_path_for_type_registry(
server.get_name(), group_name, self.get_typename())
if RepoExists(local_path):
repo = git.Repo(local_path)
if repo.is_dirty(untracked_files=True):
answer = await dirty_ui.execute(
"Worktree is dirty. Delete anyway?", "Y", "N", "C", False)
if answer:
del (repo)
DeleteLocalRepo(local_path)
else:
del (repo)
DeleteLocalRepo(local_path)
else:
DeleteLocalRepo(local_path)
async def delete_local(self, group_name: str, fail_on_dirty=False) -> bool:
"""Returns true of deleted or not there. False if deletion failed."""
server = self.get_server_routines().get_server()
local_path = self.get_server_routines().local_path_for_type_registry(
server.get_name(), group_name, self.get_typename())
if RepoExists(local_path):
repo = git.Repo(local_path)
if repo.is_dirty(untracked_files=True):
if not fail_on_dirty:
del (repo)
DeleteLocalRepo(local_path)
return True
else:
return False
else:
del (repo)
DeleteLocalRepo(local_path)
return True
else:
DeleteLocalRepo(local_path)
return True
async def pull_sub_routine(self, feedback_ui: AbstractFeedbackUI,
branch: str) -> bool:
server = self.get_server_routines().get_server()
local_repo_path = self.get_local_repo_path()
remote_url = self.get_remote_url()
await feedback_ui.output("Pulling: " + local_repo_path + " from: " +
remote_url)
if RepoExists(local_repo_path):
repo = git.Repo(local_repo_path)
remote = create_update_remote(repo, "origin", remote_url)
remote = create_update_remote(repo, server.get_name(), remote_url)
assert isinstance(remote, git.Remote)
await feedback_ui.output(("Pulling " + branch + ": " +
local_repo_path + " <- " + remote_url))
remote.pull(branch)
return True
else:
await feedback_ui.error(
"Local repository doesn't exist. Cannot pull. You might want to clone it or init it?"
)
return False
async def push_sub_routine(self, feedback_ui: AbstractFeedbackUI,
branch: str, fail_on_dirty: bool) -> bool:
"""Meant to be called by other routines.
Provides feedback.
Returns True on success, False on failure."""
server = self.get_server_routines().get_server()
remote_url = self.get_remote_url()
local_path = self.get_local_repo_path()
await feedback_ui.output("Pushing: " + local_path + " to: " +
remote_url)
if not RepoExists(local_path):
await feedback_ui.error(
"No local worktree. You might need to create or pull it first."
)
return False
repo = git.Repo(local_path)
if repo.is_dirty(index=True,
working_tree=True,
untracked_files=True,
submodules=True) and fail_on_dirty:
await feedback_ui.error("Worktree is dirty. Aborting.")
return False
# we push to the server even if there was no commit because this is a "push" command.
# this works for submodules too, we think. Maybe?
remote = create_update_remote(repo, server.get_name(), remote_url)
assert isinstance(remote, git.Remote)
await feedback_ui.output("Pushing " + branch + ": " + local_path +
" -> " + remote_url)
remote.push(branch)
return True
async def automanage_root(self, feedback_ui: AbstractFeedbackUI, root_id: str, container_id: str,
container_config: ContainerAutomanagerConfigurationComponent,
legal_entity_config: LegalEntityConfig, gitlab_server: str):
# pre automanage hook
# we call regardless of mode.
entrypoints = pkg_resources.iter_entry_points("fiepipe.plugin.automanager.pre_automanage_root")
ret = {}
for entrypoint in entrypoints:
method = entrypoint.load()
await method(feedback_ui, legal_entity_config, container_config, root_id)
# we only execute those that are configured and are checked out.
root_routines = GitRootRoutines(container_id, root_id)
root_routines.load()
await feedback_ui.output("Automanaging root: " + root_routines.root.GetName())
if not RepoExists(root_routines.get_local_repo_path()):
await feedback_ui.output("Root not checked out. Moving on.")
return
# update the root from gitlab.
# TODO: per root gitlab server?
# we update the root automatically in workstation mode.
if legal_entity_config.get_mode() == LegalEntityMode.USER_WORKSTATION:
gitlab_server_routines = GitLabServerRoutines(gitlab_server)
gitlab_routines = GitLabFQDNGitRootRoutines(gitlab_server_routines, root_routines.root,
root_routines.root_config, legal_entity_config.get_fqdn())
#does the remote exist.
exists = await gitlab_routines.remote_exists(feedback_ui)
if not exists:
#we push it up if not
await feedback_ui.output("Root doesn't exist on server. Pushing...")
success = await gitlab_routines.push_sub_routine(feedback_ui, 'master', False)
if not success:
await feedback_ui.error("Failed to push new repository. Aborting auto-management of this root")
return
else:
#if it exists, we check its ahead/behind status and act accordingly.
is_behind_remote = await gitlab_routines.is_behind_remote(feedback_ui)
is_ahead_of_remote = await gitlab_routines.is_aheadof_remote(feedback_ui)
if is_ahead_of_remote and not is_behind_remote:
await feedback_ui.output("Root is ahead. Pushing...")
success = await gitlab_routines.push_sub_routine(feedback_ui, 'master', False)
if not success:
await feedback_ui.warn(
"Failed to push commits. This is not fatal. Continuing auto-management of this root.")
if is_ahead_of_remote and is_behind_remote:
await feedback_ui.output("Root is both ahead and behind. Pulling first...")
success = await gitlab_routines.pull_sub_routine(feedback_ui, 'master')
if not success:
await feedback_ui.error("Failed to pull from remote. Aborting auto-management of this root.")
return
if not root_routines.is_in_conflict():
await feedback_ui.output("No conflicts found. Pushing...")
success = await gitlab_routines.push_sub_routine(feedback_ui, 'master', False)
if not success:
await feedback_ui.warn(
"Failed to push commits. This is not fatal. Continuing auto-management of this root.")
if not is_ahead_of_remote and is_behind_remote:
await feedback_ui.output("Root is behind. Pulling...")
success = await gitlab_routines.pull_sub_routine(feedback_ui, 'master')
if not success:
await feedback_ui.warn("Failed to pull from remote. Aborting auto-management of this root.")
return
if root_routines.is_in_conflict():
await feedback_ui.warn(
"Root is in conflict. You'll need to resolve this. Aborting auto-management of this root.")
return
# If we got here, the remote exists, we're not in conflict, and not knowingly behind.
# The worktree might be dirty though.
# However, auto adds and commits require some knowledge of structure. So we leave a dirty root to be handled
# by structure code for now.
# from here, we need to automanage root structures.
# structures are plugins.
# every plugin structure type gets an opportunity to run here. Each plugin is responsible
# for determining if it should just do nothing and return, or if it needs to do something to
# for this root. As a rule, structures are supposed to be opt-in.
entrypoints = pkg_resources.iter_entry_points("fiepipe.plugin.automanager.automanage_structure")
ret = {}
for entrypoint in entrypoints:
method = entrypoint.load()
await method(feedback_ui, root_id, container_id, container_config, legal_entity_config, gitlab_server)
async def merge_local_to_master_subroutine(self,
feedback_ui: AbstractFeedbackUI,
group_name: str):
server = self.get_server_routines().get_server()
local_path = self.get_server_routines().local_path_for_type_registry(
server.get_name(), group_name, self.get_typename())
if not RepoExists(local_path):
#nothing to merge. So, done.
return
repo = git.Repo(local_path)
local_branch = None
master_branch = None
#check for branches.
for branch in repo.heads:
assert isinstance(branch, git.Head)
if branch.name == "local":
local_branch = branch
if branch.name == "master":
master_branch = branch
if master_branch is None:
raise RuntimeError("There is no master branch.")
if local_branch is None:
#nothing to merge. So, done.
return
#checkout master
master_branch.checkout(force=True)
if repo.is_dirty():
raise RuntimeError("Checkout of master is dirty. Won't continue.")
#we merge local into master.
await feedback_ui.output("Merging local into master.")
merge_output = repo.git.merge("--no-edit", "local")
await feedback_ui.output(merge_output)
#repo.index.merge_tree(local_branch)
if is_in_conflict(repo):
raise RuntimeError(
"Conflicts found after merge. Manual resolution required. Won't continue."
)
if repo.is_dirty():
repo.index.commit("Merged local changes to master.")
#if we got here, then we succesfully merged from local to master. We merge back, to make future merges easier.
local_branch.checkout(force=True)
if repo.is_dirty():
raise RuntimeError("Checkout of local is dirty. Won't continue.")
await feedback_ui.output("Merging master back into local.")
merge_output = repo.git.merge("--no-edit", "master")
await feedback_ui.output(merge_output)
#repo.index.merge_tree(master_branch)
if is_in_conflict(repo):
raise RuntimeError(
"Conflicts found after merging master back to local. Manual resolution required. Won't continue."
)
if repo.is_dirty():
repo.index.commit("Merged master back to local.")
async def commit_to_local_subroutine(self, feedback_ui: AbstractFeedbackUI,
group_name: str):
"""Ensures the head of the local branch matches the database exactly.
Which often involves a commit."""
server = self.get_server_routines().get_server()
local_path = self.get_server_routines().local_path_for_type_registry(
server.get_name(), group_name, self.get_typename())
manager_routines = self.get_local_manager_routines()
if not RepoExists(local_path):
raise RuntimeError("No local worktree.")
repo = git.Repo(local_path)
local_branch = None
master_branch = None
#check for branches and create local if needed.
for branch in repo.heads:
assert isinstance(branch, git.Head)
if branch.name == "local":
local_branch = branch
if branch.name == "master":
master_branch = branch
if master_branch is None:
raise RuntimeError("There is no master branch.")
if local_branch is None:
await feedback_ui.output("Creating local branch from master.")
branch_output = repo.git.branch("local", "master")
await feedback_ui.output(branch_output)
await feedback_ui.output("Branch created.")
for branch in repo.heads:
assert isinstance(branch, git.Head)
if branch.name == "local":
local_branch = branch
break
if local_branch is None:
raise RuntimeError(
"No local branch even though we just created it.")
#check out local
local_branch.checkout(force=True)
#we clear the dir of items
items = os.listdir(local_path)
for item in items:
if item.endswith(".json"):
os.remove(os.path.join(local_path, item))
#then we export from the db
await manager_routines.ExportAllRoutine(local_path)
#do an add
add_output = repo.git.add("--all")
await feedback_ui.output(add_output)
#do a commit
if repo.is_dirty():
commit_output = repo.git.commit("-a", "-m",
"\"commiting db to local.\"")
await feedback_ui.output(commit_output)