def _delete_depot_paths(self):
"""If we have any depot files to delete, do so in a single changelist."""
self._log_results( "Delete commit object file(s)"
, self.del_depot_path_list
, use_preview_level = True
, enquote = True )
if not self.del_depot_path_list:
return
description = _("Git Fusion '{repo}' rollback to @{change_num}") \
.format( repo = self.ctx.config.repo_name
, change_num = self.change_num )
try:
with p4gf_util.NumberedChangelist(
gfctx=self.ctx
, description = description ) as nc:
for dp in self.del_depot_path_list:
self.p4gfrun('sync', '-k', dp)
for dp in self.del_depot_path_list:
self.p4gfrun('delete', '-k', dp)
if not self.is_preview:
nc.submit()
LOG.info("Submitted changelist: {}".format(nc.change_num))
except Exception: # pylint: disable=broad-except
LOG.exception("Could not delete commit object file(s)."
" Continuing...")
def _create_file(p4, client_name, local_path, file_content):
"""Create and submit a file.
Write a file to the local Git Fusion workspace and then add and submit to
Perforce. NOP if file already exists in Perforce after a 'p4 sync'.
"""
filename = os.path.basename(local_path)
with p4gf_util.restore_client(p4, client_name):
try:
with p4.at_exception_level(p4.RAISE_NONE):
# Sync the file and ensure we really have it.
p4.run('sync', '-q', local_path)
results = p4.run('have', local_path)
if not results:
LOG.debug("_write_file(): {} does not exist, will create...".format(local_path))
# Perms are probably read-only, need to remove before writing.
if os.path.exists(local_path):
os.remove(local_path)
else:
p4gf_util.ensure_parent_dir(local_path)
with open(local_path, 'w') as mf:
mf.write(file_content)
desc = _("Creating initial '{filename}' file via p4gf_init.py")\
.format(filename=filename)
with p4gf_util.NumberedChangelist(p4=p4, description=desc) as nc:
nc.p4run('add', local_path)
nc.submit()
LOG.debug("_write_file(): successfully created {}".format(local_path))
_info(_("File '{path}' created.").format(path=local_path))
else:
_info(_("File '{path}' already exists.").format(path=local_path))
except P4.P4Exception as e:
LOG.warning('error setting up {file} file: {e}'
.format(file=filename, e=str(e)))
def delete_branch_config(self, ctx, branch):
'''
Git user has deleted task branch - record the branch as deleted
in p4gf_config2.
'''
if branch.is_lightweight:
self.add_branch_config2(ctx)
# Mark the branch as deleted and add it to the list of deleted branches
branch.deleted = True
with p4gf_util.NumberedChangelist(
gfctx=ctx,
description=_("Deleting git branch '{0}'").format(
branch.git_branch_name)) as nc:
if self._add_branch_defs_to_p4(ctx):
nc.submit()
def _really_add_commits_to_p4(self, ctx):
"""actually run p4 add, submit to create mirror files in .git-fusion"""
desc = _("Git Fusion '{view}' copied to Git.").format(
view=ctx.config.view_name)
with p4gf_util.NumberedChangelist(gfctx=ctx, description=desc) as nc:
with Timer(ADD_SUBMIT):
LOG.debug("adding {0} commits to .git-fusion...".format(
len(self.commits.commits)))
# build list of objects to add, extracting them from git
add_files = [
self.__add_object_to_p4(ctx, go)
for go in self.commits.commits.values()
]
add_files = GitMirror.optimize_objects_to_add_to_p4(
ctx, add_files)
if not (len(add_files) or self.depot_branch_info_list
or self.branch_list):
# Avoid a blank line in output by printing something
ProgressReporter.write(
_('No Git objects to submit to Perforce'))
LOG.debug("_really_add_objects_to_p4() nothing to add...")
return
with Timer(P4_ADD):
files_added = self.add_objects_to_p4_2(ctx, add_files)
depot_branch_infos_added = \
self._add_depot_branch_infos_to_p4(ctx)
config2_added = self._add_branch_defs_to_p4(ctx)
cldfs_added = self._add_cldfs_to_p4(ctx)
with Timer(P4_SUBMIT):
if (files_added or depot_branch_infos_added
or config2_added or cldfs_added):
ProgressReporter.increment(
_('Submitting new Git commit objects to Perforce'))
r = nc.submit()
ObjectType.update_indexes(ctx, r)
else:
ProgressReporter.write(
_('No new Git objects to submit to Perforce'))
LOG.debug("ignoring empty change list...")
def __really_add_trees_to_p4(ctx, trees):
"""actually run p4 add, submit to create mirror files in .git-fusion"""
desc = _("Git Fusion '{view}' trees copied to Git.").format(
view=ctx.config.view_name)
with p4gf_util.NumberedChangelist(gfctx=ctx, description=desc) as nc:
LOG.debug("adding {} trees to .git-fusion...{}".format(
len(trees), trees))
# build list of trees to add, extracting them from git
add_files = [__add_tree_to_p4(ctx, sha1) for sha1 in list(trees)]
add_files = p4gf_gitmirror.GitMirror.optimize_objects_to_add_to_p4(
ctx, add_files)
if not len(add_files):
return
files_added = p4gf_gitmirror.GitMirror.add_objects_to_p4_2(
ctx, add_files)
if files_added:
nc.submit()
else:
LOG.debug("ignoring empty change list...")
def process_tags(ctx, tags):
"""Add or remove tags objects from the Git Fusion mirror.
:param ctx: P4GF context with initialized pygit2 Repository.
:param tags: list of PreReceiveTuple objects for tags
"""
# pylint:disable=too-many-branches
if not tags:
LOG.debug("process_tags() no incoming tags to process")
return
# Re-sync the tags since preflight_tags() synced with a different temp client.
tags_path = "objects/repos/{repo}/tags".format(repo=ctx.config.repo_name)
with ctx.p4gf.at_exception_level(P4.P4.RAISE_NONE):
# Raises an exception when there are no files to sync?
ctx.p4gfrun('sync', '-q', "//{}/{}/...".format(ctx.p4gf.client,
tags_path))
# Decide what to do with the tag references.
tags_to_delete = []
tags_to_add = []
tags_to_edit = []
for prt in tags:
tag = prt.ref[10:]
if prt.old_sha1 == p4gf_const.NULL_COMMIT_SHA1:
if prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# No idea how this happens, but it did, so guard against it.
continue
# Adding a new tag; if it references a commit, check that it
# exists; for other types, it is too costly to verify
# reachability from a known commit, so just ignore them.
obj = _get_tag_target(ctx.repo, prt.new_sha1)
is_commit = obj.type == pygit2.GIT_OBJ_COMMIT
if is_commit and not ObjectType.commits_for_sha1(
ctx, p4gf_pygit2.object_to_sha1(obj)):
LOG.debug("Tag '{}' of unknown commit {:7.7} not stored."
" Removing ref from git repo.".format(
tag, prt.new_sha1))
_remove_tag_ref(tag, prt.new_sha1)
continue
if obj.type == pygit2.GIT_OBJ_TREE:
continue
if obj.type == pygit2.GIT_OBJ_BLOB:
continue
_add_tag(ctx, tag, prt.new_sha1, tags_to_edit, tags_to_add)
elif prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# Removing an existing tag
_remove_tag(ctx, tag, prt.old_sha1, tags_to_edit, tags_to_delete)
# Seemingly nothing to do.
if not tags_to_add and not tags_to_edit and not tags_to_delete:
LOG.debug("process_tags() mysteriously came up empty"
" - probably a tag of a non-existing commit.")
return
# Add and remove tags as appropriate, doing so in batches.
LOG.info(
"adding {} tags, removing {} tags, editing {} tags from Git mirror".
format(len(tags_to_add), len(tags_to_delete), len(tags_to_edit)))
desc = _("Git Fusion '{repo}' tag changes").format(
repo=ctx.config.repo_name)
with p4gf_util.NumberedChangelist(gfctx=ctx, description=desc) as nc:
while len(tags_to_add):
bite = tags_to_add[:_BITE_SIZE]
tags_to_add = tags_to_add[_BITE_SIZE:]
ctx.p4gfrun('add', '-t', 'binary+F', bite)
while len(tags_to_edit):
bite = tags_to_edit[:_BITE_SIZE]
tags_to_edit = tags_to_edit[_BITE_SIZE:]
ctx.p4gfrun('edit', '-k', bite)
while len(tags_to_delete):
bite = tags_to_delete[:_BITE_SIZE]
tags_to_delete = tags_to_delete[_BITE_SIZE:]
ctx.p4gfrun('delete', bite)
nc.submit()
if nc.submitted:
_write_last_copied_tag(ctx, nc.change_num)
LOG.debug("process_tags() complete")
def delete_client(args, p4, client_name, metrics, prune_objs=True):
"""Delete the named Perforce client and its workspace. Raises
P4Exception if the client is not present, or the client configuration is
not set up as expected.
Keyword arguments:
args -- parsed command line arguments
p4 -- Git user's Perforce client
client_name -- name of client to be deleted
metrics -- DeletionMetrics for collecting resulting metrics
prune_objs -- if True, delete associated objects from cache
"""
# pylint: disable=R0912,R0915
group_list = [
p4gf_const.P4GF_GROUP_VIEW_PULL, p4gf_const.P4GF_GROUP_VIEW_PUSH
]
p4.user = p4gf_const.P4GF_USER
print_verbose(args, _("Checking for client '{}'...").format(client_name))
if not p4gf_util.spec_exists(p4, 'client', client_name):
raise P4.P4Exception(
_("No such client '{}' defined").format(client_name))
view_name = p4gf_util.client_to_view_name(client_name)
p4gf_dir = p4gf_util.p4_to_p4gf_dir(p4)
view_dirs = p4gf_view_dirs.from_p4gf_dir(p4gf_dir, view_name)
p4gf_util.ensure_spec_values(p4, 'client', client_name,
{'Root': view_dirs.p4root})
view_lock = None # We're clobbering and deleting. Overrule locks.
with p4gf_context.create_context(view_name, view_lock) as ctx:
command_path = ctx.client_view_path()
homedir = os.path.expanduser('~')
raise_if_homedir(homedir, view_name, view_dirs.view_container)
# Scan for objects associated only with this view so we can remove them.
objects_to_delete = []
if prune_objs:
objects_to_delete = _find_client_commit_objects(
args, p4, view_name)
# Do we have a repo config file to delete?
config_file = p4gf_config.depot_path_repo(view_name) + '*'
config_file_exists = p4gf_util.depot_file_exists(p4, config_file)
# What counters shall we delete?
counter_list = []
counter_list.append(
p4gf_context.calc_last_copied_change_counter_name(
view_name, p4gf_util.get_server_id()))
for spec in p4.run('counters', '-u', '-e',
"git-fusion-index-last-{},*".format(view_name)):
counter_list.append(spec['counter'])
for spec in p4.run('counters', '-u', '-e',
"git-fusion-index-branch-{},*".format(view_name)):
counter_list.append(spec['counter'])
if not args.delete:
print(NTR('p4 sync -f {}#none').format(command_path))
print(NTR('p4 client -f -d {}').format(client_name))
print(NTR('rm -rf {}').format(view_dirs.view_container))
print(
NTR('Deleting {} objects from //{}/objects/...').format(
len(objects_to_delete), p4gf_const.P4GF_DEPOT))
for group_template in group_list:
group = group_template.format(view=view_name)
print(NTR('p4 group -a -d {}').format(group))
for c in counter_list:
print(NTR('p4 counter -u -d {}').format(c))
if config_file_exists:
print(NTR('p4 sync -f {}').format(config_file))
print(NTR('p4 delete {}').format(config_file))
print(
NTR('p4 submit -d "Delete repo config for {view_name}" {config_file}'
).format(view_name=view_name, config_file=config_file))
else:
print_verbose(
args,
NTR('Removing client files for {}...').format(client_name))
ctx.p4.run('sync', '-fq', command_path + '#none')
print_verbose(args,
NTR('Deleting client {}...').format(client_name))
p4.run('client', '-df', client_name)
metrics.clients += 1
print_verbose(
args,
NTR("Deleting repo {0}'s directory {1}...").format(
view_name, view_dirs.view_container))
_remove_tree(view_dirs.view_container, contents_only=False)
metrics.files += _delete_files(p4, objects_to_delete, view_name)
for group_template in group_list:
_delete_group(args, p4, group_template.format(view=view_name),
metrics)
for c in counter_list:
_delete_counter(p4, c, metrics)
if config_file_exists:
p4gf_util.p4run_logged(p4, ['sync', '-fq', config_file])
with p4gf_util.NumberedChangelist(
p4=p4,
description=_("Delete repo config for '{}'").format(
view_name)) as nc:
nc.p4run(["delete", config_file])
nc.submit()
def process_tags(ctx, prl):
"""
For each tag reference in the pre-receive-tuple list, add or remove the
object from the Git Fusion mirror.
Arguments:
ctx - P4GF context with initialized pygit2 Repository.
prl - list of PreReceiveTuple objects.
Returns None if successful and an error string otherwise.
"""
tags = [prt for prt in prl if prt.ref.startswith('refs/tags/')]
if not tags:
LOG.debug("process_tags() no incoming tags to process")
return None
# Screen the tags to ensure their names won't cause problems sometime
# in the future (i.e. when we create Perforce labels). Several of these
# characters are not allowed in Git tag names anyway, but better to
# check in case that changes in the future.
# In particular git disallows a leading '-', but we'll check for it anyway
# Otherwise allow internal '-'
regex = re.compile(r'[*@#,]|\.\.\.|%%')
for prt in tags:
tag = prt.ref[10:]
if regex.search(tag) or tag.startswith('-'):
return _(
"illegal characters (@#*,...%%) in tag name: '{}'").format(tag)
if not ctx.view_repo:
# In some cases the Git repository object is not yet created.
ctx.view_repo = pygit2.Repository(ctx.view_dirs.GIT_DIR)
LOG.debug("process_tags() beginning...")
tags_path = "objects/repos/{repo}/tags/...".format(
repo=ctx.config.view_name)
with ctx.p4gf.at_exception_level(P4.P4.RAISE_NONE):
# Raises an exception when there are no files to sync?
ctx.p4gfrun([
'sync', '-q', "//{}/{}/...".format(ctx.config.p4client_gf,
tags_path)
])
# Decide what to do with the tag references.
tags_to_delete = []
tags_to_add = []
tags_to_edit = []
for prt in tags:
tag = prt.ref[10:]
if prt.old_sha1 == p4gf_const.NULL_COMMIT_SHA1:
if prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# No idea how this happens, but it did, so guard against it.
sys.stderr.write(
_('Ignoring double-zero pre-receive-tuple line'))
continue
# Adding a new tag; if it references a commit, check that it
# exists; for other types, it is too costly to verify
# reachability from a known commit, so just ignore them.
obj = _get_tag_target(ctx.view_repo, prt.new_sha1)
is_commit = obj.type == pygit2.GIT_OBJ_COMMIT
if is_commit and not ObjectType.commits_for_sha1(ctx, obj.hex):
return _("Tag '{}' references unknown objects."
" Push commits before tags.").format(tag)
if obj.type == pygit2.GIT_OBJ_TREE:
sys.stderr.write(
_("Tag '{}' of tree will not be stored in Perforce\n").
format(tag))
continue
if obj.type == pygit2.GIT_OBJ_BLOB:
sys.stderr.write(
_("Tag '{}' of blob will not be stored in Perforce\n").
format(tag))
continue
_add_tag(ctx, tag, prt.new_sha1, tags_to_edit, tags_to_add)
elif prt.new_sha1 == p4gf_const.NULL_COMMIT_SHA1:
# Removing an existing tag
_remove_tag(ctx, tag, prt.old_sha1, tags_to_edit, tags_to_delete)
else:
# Older versions of Git allowed moving a tag reference, while
# newer ones seemingly do not. We will take the new behavior as
# the correct one and reject such changes.
return _(
'Updates were rejected because the tag already exists in the remote.'
)
# Seemingly nothing to do.
if not tags_to_add and not tags_to_edit and not tags_to_delete:
LOG.debug("process_tags() mysteriously came up empty")
return None
# Add and remove tags as appropriate, doing so in batches.
LOG.info("adding {} tags, removing {} tags from Git mirror".format(
len(tags_to_add), len(tags_to_delete)))
desc = _("Git Fusion '{repo}' tag changes").format(
repo=ctx.config.view_name)
with p4gf_util.NumberedChangelist(gfctx=ctx, description=desc) as nc:
while len(tags_to_add):
bite = tags_to_add[:_BITE_SIZE]
tags_to_add = tags_to_add[_BITE_SIZE:]
ctx.p4gfrun(["add", "-t", "binary+F", bite])
while len(tags_to_edit):
bite = tags_to_edit[:_BITE_SIZE]
tags_to_edit = tags_to_edit[_BITE_SIZE:]
ctx.p4gfrun(["edit", "-k", bite])
while len(tags_to_delete):
bite = tags_to_delete[:_BITE_SIZE]
tags_to_delete = tags_to_delete[_BITE_SIZE:]
ctx.p4gfrun(["delete", bite])
nc.submit()
if nc.submitted:
_write_last_copied_tag(ctx, nc.change_num)
LOG.debug("process_tags() complete")
return None