def extract_key_data(p4, depot_path, rev=None):
"""For the given depot path, extract the user name, SSH key, and the key
fingerprint generated from that key, returning them as a tuple. Any of
the returned values may be None if the data is missing (e.g. if path is
malformed, then user cannot be determined; if key file is malformed, no
key; likewise for the fingerprint).
"""
user = None
m = KEYPATH_RE.search(depot_path)
if m:
user = m.group(1)
fp = None
if rev:
depot_path = "{}#{}".format(depot_path, rev)
# Read all key files as raw bytes, assume they are encoded in UTF-8.
# Git Fusion does not support other encodings for key file content.
b = p4gf_util.print_depot_path_raw(p4, depot_path)
s = b.decode() # as UTF-8
lines = s.splitlines()
lines = [line for line in lines if len(line) > 0] # Strip blank lines
key = read_key_data(lines)
if key:
fp = ssh_key_to_fingerprint(key)
return (user, key, fp)
def _calc_repairs(ctx):
"""
Scan Perforce for Git commit data and Perforce changelist descriptions,
calculate which Perforce changelists need more data copied from Git
backing store //.git-fusion/objects/...
"""
# Load repo's entire set of Commit/Changelist metadata
# into memory.
LOG.info("Fetching list of Git commits/changelists from %s/objects/...",
p4gf_const.objects_root())
r = ctx.p4run(
'files', '{root}/repos/{repo}/commits/...'.format(
root=p4gf_const.objects_root(), repo=ctx.config.repo_name))
# 'p4 print' each Git commit from its backup in
# //.git-fusion/objects/...
LOG.info("Fetched commit objects: {ct}".format(ct=len(r)))
for rr in r:
depot_path = rr.get('depotFile')
if not depot_path:
continue
ot = ObjectType.commit_from_filepath(depot_path)
SHA1_TO_OTL[ot.sha1].append(ot)
LOG.debug('p4 print {}'.format(depot_path))
blob_raw = p4gf_util.print_depot_path_raw(ctx.p4, depot_path)
blob = p4gf_util.bytes_to_git_object(blob_raw)
par_list = commit_to_parent_list(blob)
SHA1_TO_PAR_SHA1_LIST[ot.sha1] = par_list
LOG.debug("{sha1:7.7} parents={par}".format(
sha1=ot.sha1, par=[p4gf_util.abbrev(p) for p in par_list]))
# Loop through changelists, comparing against
# backup and calculating if additional data
# needs to be copied to its changelist description.
return _calc_repairs_loop(ctx)
def _print_config_file(p4, depot_path):
'''
Return a config file's content as a string.
'''
b = p4gf_util.print_depot_path_raw(p4, depot_path)
if b:
return b.decode() # as UTF-8
else:
return None
def p4print(self, lfsfs):
"""Return a byte array that contains the large file content
addressed by the LFSFileSpec.
It is a programming error to call this on a non-existent or
deleted-at-head depot file.
"""
return p4gf_util.print_depot_path_raw(self.ctx.p4gf,
lfsfs.depot_path(self.ctx))
def _print_config_file(p4, depot_path):
'''
Return a config file's content as a string.
'''
b = p4gf_util.print_depot_path_raw(p4, depot_path)
if b:
return b.decode() # as UTF-8
else:
return None
def _add_branch_defs_to_p4(self, ctx):
'''
If we defined any new named+lightweight branches, update (or write the
first revision of) this repo's p4gf_config2 file with all the
currently defined named+lightweight branches.
'''
# Nothing to write? well maybe we have just deleted the remaining refs
have_branches = bool(self.branch_list)
# What does the file look like now?
p4 = ctx.p4gf # For less typing later.
old_content = None
new_content = None
depot_path = p4gf_config.depot_path_repo2(ctx.config.view_name)
local_path = p4gf_util.depot_to_local_path(depot_path, p4)
depot_exists = False
# 'p4 print' will fail if file doesn't exist yet. Okay.
with ctx.p4gf.at_exception_level(p4.RAISE_NONE):
b = p4gf_util.print_depot_path_raw(p4, depot_path)
if b:
old_content = b.decode() # as UTF-8
depot_exists = True
# What do we want the file to look like? ConfigParser writes only to
# file, not to string, so we have to give it a file path. Ooh! I know!
# How about writing to the very file that we have to 'p4 add' or 'p4
# edit' if its content differs?
if have_branches:
config = configparser.ConfigParser(interpolation=None)
for b in self.branch_list:
LOG.debug("add branch {0}".format(b))
b.add_to_config(config)
p4gf_util.ensure_dir(p4gf_util.parent_dir(local_path))
p4gf_util.make_writable(local_path)
with open(local_path, 'w') as f:
config.write(f)
with open(local_path, 'r') as f:
new_content = f.read()
p4gf_config.clean_up_parser(config)
del config
# Did nothing change? Then nothing to write.
if p4gf_config.compare_configs_string(old_content, new_content):
LOG.debug("No change to p4gf_config2 file")
return False
# Have to add or edit or delete the file.
if not have_branches:
ctx.p4gfrun(['sync', '-fkq', depot_path])
ctx.p4gfrun(['delete', depot_path])
LOG.debug("Deleted p4gf_config2 file")
else:
if depot_exists:
ctx.p4gfrun(['sync', '-fkq', depot_path])
ctx.p4gfrun(['edit', depot_path])
LOG.debug("Edited p4gf_config2 file")
else:
ctx.p4gfrun(['add', '-t', 'text', local_path])
LOG.debug("Added p4gf_config2 file")
return True
def _add_branch_defs_to_p4(self, ctx):
'''
If we defined any new named+lightweight branches, update (or write the
first revision of) this repo's p4gf_config2 file with all the
currently defined named+lightweight branches.
'''
# Nothing to write? well maybe we have just deleted the remaining refs
have_branches = bool(self.branch_list)
# What does the file look like now?
p4 = ctx.p4gf # For less typing later.
old_content = None
new_content = None
depot_path = p4gf_config.depot_path_repo2(ctx.config.view_name)
local_path = p4gf_util.depot_to_local_path(depot_path, p4)
depot_exists = False
# 'p4 print' will fail if file doesn't exist yet. Okay.
with ctx.p4gf.at_exception_level(p4.RAISE_NONE):
b = p4gf_util.print_depot_path_raw(p4, depot_path)
if b:
old_content = b.decode() # as UTF-8
depot_exists = True
# What do we want the file to look like? ConfigParser writes only to
# file, not to string, so we have to give it a file path. Ooh! I know!
# How about writing to the very file that we have to 'p4 add' or 'p4
# edit' if its content differs?
if have_branches:
config = configparser.ConfigParser(interpolation=None)
for b in self.branch_list:
LOG.debug("add branch {0}".format(b))
b.add_to_config(config)
p4gf_util.ensure_dir(p4gf_util.parent_dir(local_path))
p4gf_util.make_writable(local_path)
with open(local_path, 'w') as f:
config.write(f)
with open(local_path, 'r') as f:
new_content = f.read()
p4gf_config.clean_up_parser(config)
del config
# Did nothing change? Then nothing to write.
if p4gf_config.compare_configs_string(old_content, new_content):
LOG.debug("No change to p4gf_config2 file")
return False
# Have to add or edit or delete the file.
if not have_branches:
ctx.p4gfrun(['sync', '-fkq', depot_path])
ctx.p4gfrun(['delete', depot_path])
LOG.debug("Deleted p4gf_config2 file")
else:
if depot_exists:
ctx.p4gfrun(['sync', '-fkq', depot_path])
ctx.p4gfrun(['edit', depot_path])
LOG.debug("Edited p4gf_config2 file")
else:
ctx.p4gfrun(['add', '-t', 'text', local_path])
LOG.debug("Added p4gf_config2 file")
return True
def add_cl(self, *, branch, p4change):
"""Find .gitattributes files on branch@p4change and cache any lfs lines."""
# pylint:disable=too-many-branches
LOG.debug('add_cl {} on {}'.format(p4change.change, branch.branch_id))
# if first change on this branch, add dict change# -> P4ChangeAttribute
branch_name = branch.git_branch_name
if branch_name in self.change_gitattributes_dict:
# not the first change on this branch
if p4change.change in self.change_gitattributes_dict[branch_name]:
raise RuntimeError(
_("Change {change} seen more than once in branch {branch}"
).format(change=p4change.change, branch=branch_name))
# find the parent change and its .gitattributes
prev_change = max(
self.change_gitattributes_dict[branch_name].keys())
init_dict = self.change_gitattributes_dict[branch_name][prev_change]\
.dirpath_attributes_dict
else:
# first change on this branch
self.change_gitattributes_dict[branch_name] = {}
# first, use git-lfs-initial-track config option, if present
init_dict = {}
initial_content = p4gf_lfs_attributes.generate_initial_lfs_attrs(
self.ctx)
if initial_content:
init_dict[''] = parse_gitattributes(initial_content)
# then add in any .gitattributes in effect prior to this change
r = self.ctx.p4run(
'files',
'//.../.gitattributes@{}'.format(int(p4change.change) - 1))
for change_file in r:
depot_path = change_file['depotFile']
gwt_dir = _get_gwt_path(self.ctx, depot_path, branch,
p4change.change)
init_dict[gwt_dir] = parse_gitattributes(
p4gf_util.print_depot_path_raw(self.ctx.p4, depot_path,
p4change.change))
prev_change = p4change.change - 1
self.change_gitattributes_dict[branch_name][prev_change] = \
P4ChangeAttribute(prev_change, init_dict)
# get .gitattributes delta for p4change
change_dict = {}
for change_file in p4change.files:
depot_path = change_file.depot_path
if not depot_path.endswith('/.gitattributes'):
continue
is_delete = change_file.action == 'delete'
gwt_dir = _get_gwt_path(self.ctx, depot_path, branch,
p4change.change)
# If top level .gitattributes in first change doesn't exist in P4,
# that means we inserted it with contents of initial tracking.
# Since we took care of that above, skip it here.
if (len(self.change_gitattributes_dict[branch_name]) == 1
and gwt_dir == '' and not is_delete and
not p4gf_util.depot_file_exists(self.ctx.p4, depot_path)):
continue
# When .gitattributes is deleted, we want to remove the path from
# the dict rather than set it to an empty pattern list.
if is_delete:
if gwt_dir in change_dict:
del init_dict[gwt_dir]
else:
change_dict[gwt_dir] = parse_gitattributes(
p4gf_util.print_depot_path_raw(self.ctx.p4, depot_path,
p4change.change))
# if nothing changed, just ref the previous dict, saving a bit of memory
# otherwise apply this change's delta to the previous changes attrs
if change_dict:
updated_dict = copy.deepcopy(init_dict)
updated_dict.update(change_dict)
else:
updated_dict = init_dict
self.change_gitattributes_dict[branch_name][p4change.change] = \
P4ChangeAttribute(p4change.change, updated_dict)