def outputStat(self, h):
"""save path of current file"""
self.flush()
self.rev = P4File.create_from_print(h)
#self.change_set.add(self.rev.change)
# Not sure I want to bump this for every single file
# revision. That's frequent enough to slow down this
# print phase.
#ProgressReporter.increment('MC Copying file revisions')
# This doubles our log output. Merge this with flush(),
# uncomment only when you need to debug PrintHandler itself.
#
# Spaces here to align depot path with flush() Printed.
#_debug3( 'PrintHandler.outputStat() ch={:<5}'
# ' {}#{}'
# , self.rev.change
# , self.rev.depot_path
# , self.rev.revision )
if self.tempfile:
self.tempfile.seek(0)
self.tempfile.truncate()
else:
self.tempfile = tempfile.TemporaryFile(buffering=10000000, dir=self.tempdir,
prefix='p2g-print-')
return OutputHandler.HANDLED
def _fstat(self):
"""run fstat to find deleted revs and get client paths"""
# TODO for 12.2 print will also report deleted revs so between
# that and using MapApi to get client paths, we won't need this fstat
self.ctx.p4.handler = FstatHandler(self.printed_revs, self.changes)
fstat_cols = "-T" + ",".join(P4File.fstat_cols())
self.ctx.p4.run("fstat", "-Of", fstat_cols, self._path_range())
if self.graft_change:
# Also run 'p4 fstat //<view>/...@change' for the graft
# change to catch all files as of @change, not just
# revs changed between begin and end of _path_range().
self.ctx.p4.run("fstat", fstat_cols, self._graft_path())
self.ctx.p4.handler = None
self._collapse_to_graft_change()
self._add_graft_to_changes()
# don't need this any more
self.printed_revs = None
sorted_changes = [str(y) for y in sorted([int(x) for x in self.changes.keys()])]
LOG.debug("\n".join([str(self.changes[ch]) for ch in sorted_changes]))
return sorted_changes
def _fstat(self):
"""run fstat to find deleted revs and get client paths"""
# TODO for 12.2 print will also report deleted revs so between
# that and using MapApi to get client paths, we won't need this fstat
self.ctx.p4.handler = FstatHandler(self.printed_revs, self.changes)
fstat_cols = "-T" + ",".join(P4File.fstat_cols())
self.ctx.p4.run("fstat", "-Of", fstat_cols, self._path_range())
if self.graft_change:
# Also run 'p4 fstat //<view>/...@change' for the graft
# change to catch all files as of @change, not just
# revs changed between begin and end of _path_range().
self.ctx.p4.run("fstat", fstat_cols, self._graft_path())
self.ctx.p4.handler = None
self._collapse_to_graft_change()
self._add_graft_to_changes()
# don't need this any more
self.printed_revs = None
sorted_changes = [
str(y) for y in sorted([int(x) for x in self.changes.keys()])
]
LOG.debug("\n".join([str(self.changes[ch]) for ch in sorted_changes]))
return sorted_changes
def outputStat(self, h):
"""save path of current file"""
self.flush()
self.rev = P4File.create_from_print(h)
#self.change_set.add(self.rev.change)
# Not sure I want to bump this for every single file
# revision. That's frequent enough to slow down this
# print phase.
#ProgressReporter.increment('MC Copying file revisions')
# This doubles our log output. Merge this with flush(),
# uncomment only when you need to debug PrintHandler itself.
#
# Spaces here to align depot path with flush() Printed.
#_debug3( 'PrintHandler.outputStat() ch={:<5}'
# ' {}#{}'
# , self.rev.change
# , self.rev.depot_path
# , self.rev.revision )
if self.tempfile:
self.tempfile.seek(0)
self.tempfile.truncate()
else:
self.tempfile = tempfile.TemporaryFile(buffering=10000000,
dir=self.tempdir,
prefix='p2g-print-')
return OutputHandler.HANDLED
def maybe_create_lfs_attrs(ctx, changelist, p4file_list, branch):
"""Create the initial .gitattributes file for an LFS-enabled repo.
Does nothing if the git-lfs-initial-track configurable is not set.
:param ctx: Git Fusion context.
:param changelist: the P4Changelist.
:param p4file_list: list of P4File.
:param branch: the Branch to contain this file.
:return: modified p4file_list
"""
# Check if we have an initial-tracking setting or not.
initial_tracking = generate_initial_lfs_attrs(ctx)
if not initial_tracking:
return p4file_list
# Check if a .gitattributes file already exists or not.
with ctx.switched_to_branch(branch):
depot_path = ctx.gwt_to_depot_path(p4gf_const.GITATTRIBUTES)
if not depot_path:
return p4file_list
for p4file in p4file_list:
if p4file.depot_path == depot_path:
# A .gitattributes already exists, nothing to do.
return p4file_list
# If a .gitattributes file ever existed in Perforce but was deleted by
# the time we got to this changelist, honor that deletion. Do not insert.
r = ctx.p4run('files', "{}@{}".format(depot_path, changelist.change))
if p4gf_util.first_dict_with_key(r, "depotFile"):
return p4file_list
# "Print" the attributes file into the repository.
sha1 = add_attributes_to_repo(initial_tracking, ctx.repo)
if sha1 is None:
return p4file_list
# Save the SHA1 in the gitmirror list of pending blobs to cache.
ctx.mirror.add_blob(sha1)
# Construct a P4File and add to the list of files for this change.
vardict = {
'depotFile': depot_path,
'action': 'add',
'rev': 1,
'type': 'text',
'change': changelist.change
}
p4file = P4File.create_from_print(vardict)
p4file.sha1 = sha1
p4file_list.append(p4file)
return p4file_list
def outputStat(self, h):
"""grab clientFile from fstat output"""
p4file = P4File.create_from_sync(h)
if p4file.is_delete():
if os.path.exists(p4file.client_path):
os.unlink(p4file.client_path)
else:
if not os.path.exists(p4file.client_path):
if not os.path.exists(os.path.dirname(p4file.client_path)):
os.makedirs(os.path.dirname(p4file.client_path))
with open(p4file.client_path, 'a'):
pass
return OutputHandler.HANDLED
def outputStat(self, h):
"""grab clientFile from fstat output"""
p4file = P4File.create_from_sync(h)
if p4file.is_delete():
if os.path.exists(p4file.client_path):
os.unlink(p4file.client_path)
else:
if not os.path.exists(p4file.client_path):
if not os.path.exists(os.path.dirname(p4file.client_path)):
os.makedirs(os.path.dirname(p4file.client_path))
with open(p4file.client_path, 'a'):
pass
return OutputHandler.HANDLED
def outputStat(self, h):
"""save path of current file"""
self.flush()
self.rev = P4File.create_from_print(h)
self.change_set.add(self.rev.change)
ProgressReporter.increment(_('Copying files'))
LOG.debug2("PrintHandler.outputStat() ch={} {}#{}".format(
self.rev.change, self.rev.depot_path, self.rev.revision))
# use the git working tree so we can use create_blob_fromfile()
tmpdir = os.getcwd()
self.tempfile = tempfile.NamedTemporaryFile(
buffering=10000000, prefix='p2g-print-', dir=tmpdir, delete=False)
return OutputHandler.HANDLED
def outputStat(self, h):
"""save path of current file"""
self.flush()
self.rev = P4File.create_from_print(h)
self.revs.append(self.rev)
self.progress.progress_increment('Copying files')
LOG.debug("PrintHandler.outputStat() ch={} {}"
.format(h['change'], h["depotFile"] + "#" + h["rev"]))
if self.tempfile:
self.tempfile.seek(0)
self.tempfile.truncate()
else:
self.tempfile = tempfile.TemporaryFile(buffering=10000000, dir=self.tempdir)
return OutputHandler.HANDLED
def outputStat(self, h):
"""grab clientFile from fstat output"""
key = h["depotFile"] + "#" + h["headRev"]
p4file = self.revs.find(key)
if p4file:
p4file.client_path = h["clientFile"]
else:
# deleted files not reported by p4 print
p4file = P4File.create_from_fstat(h)
# ignore any deletions that happened before our starting change
if not p4file.change in self.changes:
LOG.debug("skipping deleted rev:{}".format(p4file.rev_path()))
return OutputHandler.HANDLED
self.changes[p4file.change].files.append(p4file)
return OutputHandler.HANDLED
def outputStat(self, h):
"""grab clientFile from fstat output"""
key = h["depotFile"] + "#" + h["headRev"]
p4file = self.revs.find(key)
if p4file:
p4file.client_path = h["clientFile"]
else:
# deleted files not reported by p4 print
p4file = P4File.create_from_fstat(h)
# ignore any deletions that happened before our starting change
if not p4file.change in self.changes:
LOG.debug("skipping deleted rev:{}".format(p4file.rev_path()))
return OutputHandler.HANDLED
self.changes[p4file.change].files.append(p4file)
return OutputHandler.HANDLED
def outputStat(self, h):
"""save path of current file"""
self.flush()
self.rev = P4File.create_from_print(h)
self.revs.append(self.rev)
self.progress.progress_increment('Copying files')
LOG.debug("PrintHandler.outputStat() ch={} {}".format(
h['change'], h["depotFile"] + "#" + h["rev"]))
if self.tempfile:
self.tempfile.seek(0)
self.tempfile.truncate()
else:
self.tempfile = tempfile.TemporaryFile(buffering=10000000,
dir=self.tempdir)
return OutputHandler.HANDLED
def create_using_describe(p4, change, depot_root):
"""create a P4Changelist by running p4 describe"""
result = p4.run("describe", "-s", str(change))
cl = P4Changelist()
vardict = p4gf_util.first_dict_with_key(result, 'change')
cl.change = vardict["change"]
cl.description = vardict["desc"]
cl.user = vardict["user"]
cl.time = vardict["time"]
for i in range(len(vardict["depotFile"])):
p4file = P4File.create_from_describe(vardict, i)
# filter out files not under our root right now
if not p4file.depot_path.startswith(depot_root):
continue
cl.files.append(p4file)
return cl
def create_using_describe(p4, change, depot_root):
"""create a P4Changelist by running p4 describe"""
result = p4.run("describe", "-s", str(change))
cl = P4Changelist()
vardict = p4gf_util.first_dict_with_key(result, "change")
cl.change = vardict["change"]
cl.description = vardict["desc"]
cl.user = vardict["user"]
cl.time = vardict["time"]
for i in range(len(vardict["depotFile"])):
p4file = P4File.create_from_describe(vardict, i)
# filter out files not under our root right now
if not p4file.depot_path.startswith(depot_root):
continue
cl.files.append(p4file)
return cl
def outputStat(self, h):
"""Save path of current file."""
try:
self.flush()
self.rev = P4File.create_from_print(h)
self.change_set.add(self.rev.change)
ProgressReporter.increment(_('Copying files'))
LOG.debug2("PrintHandler.outputStat() ch={} {}#{}".format(
self.rev.change, self.rev.depot_path, self.rev.revision))
# use the git working tree so we can use create_blob_fromfile()
tmpdir = os.getcwd()
self.temp_file = tempfile.NamedTemporaryFile(buffering=10000000,
prefix='p2g-print-',
dir=tmpdir,
delete=False)
LOG.debug3('outputStat() temporary file created: {}'.format(
self.temp_file.name))
except Exception: # pylint: disable=broad-except
LOG.exception("outputStat")
return OutputHandler.HANDLED
def _copy_one(self, cnob):
'''
Copy one ChangeNumOnBranch element from Perforce to Git.
p4 print all of its file revisions directly into .git/objects as blobs
add them to the git-fast-import script
'''
_debug2('_copy_one {}', cnob)
branch = self.ctx.branch_dict().get(cnob.branch_id)
with self.ctx.switched_to_branch(branch):
# Keep track of the highest changelist number we've
# copied. Can't rely on
# self.change_num_on_branch_list[-1] starting with our
# highest changelist number because we might discover
# new branches during later calls to _copy_one().
change_num = int(cnob.change_num)
if self.highest_copied_change_num < change_num:
self.highest_copied_change_num = change_num
self.cnob_count += 1
# p4 changes -l -m1 @nnn
#
# Gets changelist description (including possible DescInfo),
# owner, time.
with self.p2g.perf.timer[CHANGES1]:
r = self.ctx.p4run([
'changes',
'-l' # include full changelist description
,
'-m1' # just this one changelist
,
'@{}'.format(cnob.change_num)
])
p4changelist = P4Changelist.create_using_changes(r[0])
# p4 filelog -c nnnn -m1 //change_path/...
#
# Gets integration sources for parent calculations.
# Gets files deleted at this rev (which 'p4 print' won't on 11.1).
# Gets file list for this changelist.
#
# Cannot use p4 filelog //{client}/...@=nnn
# That does request does not return one fstat for each file
# in changelist nnn.
with self.p2g.perf.timer[FILELOG]:
cmd = ['filelog', '-c', cnob.change_num, '-m1', cnob.path]
filelog_results = self.ctx.p4run(cmd)
### Detect lightweight integration sources not yet known.
### Create new Branch views to map them into this repo,
### run 'p4 changes' on them to add their history to our
### change_num_on_branch_list work queue, sorted.
dbil = self.p2g.to_depot_branch_list_mc(change_num,
filelog_results)
new_dbi_set = set(dbil) - self.known_dbi_set
if new_dbi_set:
### push_front cnob
for dbi in new_dbi_set:
LOG.error('AHA detected new integ source: {}'.format(dbi))
### process dbi into branch, branch into more cnobs
### mergesort new cnobs into cnob deque
self.known_dbi_set.update(new_dbi_set)
### return, we'll deal with this later
### +++ save changes and filelog work
# p4 print every revision modified by this changelist.
#
# +++ Also print every revision AFTER this changelist. There's a
# +++ high probability that we'll need those revisons later.
# +++ Printing them all now _greatly_ reduces the total number of
# +++ 'p4 print' requests, reduces the Perforce server's workload
# +++ (and thus repsponse time) in generating incremental file
# +++ revisions from any files stored using RCS deltas (aka most
# +++ files).
with self.p2g.perf.timer[CALC_PRINT]:
depot_path_rev_list = []
for rr in filelog_results:
if ((not isinstance(rr, dict)) or ('depotFile' not in rr)
or ('rev' not in rr)):
continue
p4file = P4File.create_from_filelog(rr)
p4changelist.files.append(p4file)
depot_path = rr['depotFile']
rev = rr['rev'][0]
if self._already_printed(depot_path, rev):
continue
depot_path_rev_list.append('{}#{},head'.format(
depot_path, rev))
rev_total = len(p4changelist.files)
_debug2(
'Printing files.'
' change: {change_num}'
' total: {rev_total}'
' need_print: {rev_need_print}'
' already_printed: {rev_already_printed}',
change_num=cnob.change_num,
rev_need_print=len(depot_path_rev_list),
rev_already_printed=rev_total - len(depot_path_rev_list),
rev_total=rev_total)
if depot_path_rev_list:
with self.p2g.perf.timer[PRINT2]:
printhandler = self._print_handler()
server_can_unexpand = self.ctx.p4.server_level > 32
args = ["-a"]
if server_can_unexpand:
args.append("-k")
cmd = ['print'] + args + depot_path_rev_list
with p4gf_util.RawEncoding(self.ctx.p4) \
, p4gf_util.Handler(self.ctx.p4, printhandler) \
, self.ctx.p4.at_exception_level(P4.RAISE_ALL):
self.ctx.p4run(cmd)
printhandler.flush()
# Find each file revision's blob sha1.
for p4file in p4changelist.files:
symlink_path = _depot_rev_to_symlink(
depot_path=p4file.depot_path,
rev=p4file.revision,
symlink_dir=self.symlink_dir)
blob_path = os.readlink(symlink_path)
p4file.sha1 = _blob_path_to_sha1(blob_path)
# If we can copy the Git commit and its tree objects from
# our gitmirror, do so.
# Non-MemCapped code calls all FI functions with
# timer[FAST_IMPORT] as outer container, so must we.
with self.p2g.perf.timer[FAST_IMPORT]:
if self.p2g._fast_import_from_gitmirror(p4changelist, branch):
LOG.debug2('@{} fast-imported from gitmirror.'.format(
cnob.change_num))
return
# Build a git-fast-import commit object.
### _fast_import_from_p4() runs its own filelog to
### discover integ sources. That needs to be hoisted up
### to our own filelog and passed down to avoid
### duplicate work.
LOG.debug2('@{} fast-importing from p4 changelist.'.format(
cnob.change_num))
self.p2g._fast_import_from_p4_mc(
change=p4changelist,
branch=branch,
filelog_results=filelog_results,
mark_to_branch_id=self.mark_to_branch_id,
branch_id_to_temp_name=self.branch_id_to_temp_name)
def _copy_one(self, cnob):
'''
Copy one ChangeNumOnBranch element from Perforce to Git.
p4 print all of its file revisions directly into .git/objects as blobs
add them to the git-fast-import script
'''
_debug2('_copy_one {}', cnob)
branch = self.ctx.branch_dict().get(cnob.branch_id)
with self.ctx.switched_to_branch(branch):
# Keep track of the highest changelist number we've
# copied. Can't rely on
# self.change_num_on_branch_list[-1] starting with our
# highest changelist number because we might discover
# new branches during later calls to _copy_one().
change_num = int(cnob.change_num)
if self.highest_copied_change_num < change_num:
self.highest_copied_change_num = change_num
self.cnob_count += 1
# p4 changes -l -m1 @nnn
#
# Gets changelist description (including possible DescInfo),
# owner, time.
with self.p2g.perf.timer[CHANGES1]:
r = self.ctx.p4run([ 'changes'
, '-l' # include full changelist description
, '-m1' # just this one changelist
, '@{}'.format(cnob.change_num)])
p4changelist = P4Changelist.create_using_changes(r[0])
# p4 filelog -c nnnn -m1 //change_path/...
#
# Gets integration sources for parent calculations.
# Gets files deleted at this rev (which 'p4 print' won't on 11.1).
# Gets file list for this changelist.
#
# Cannot use p4 filelog //{client}/...@=nnn
# That does request does not return one fstat for each file
# in changelist nnn.
with self.p2g.perf.timer[FILELOG]:
cmd = [ 'filelog'
, '-c', cnob.change_num
, '-m1'
, cnob.path]
filelog_results = self.ctx.p4run(cmd)
### Detect lightweight integration sources not yet known.
### Create new Branch views to map them into this repo,
### run 'p4 changes' on them to add their history to our
### change_num_on_branch_list work queue, sorted.
dbil = self.p2g.to_depot_branch_list_mc(change_num, filelog_results)
new_dbi_set = set(dbil) - self.known_dbi_set
if new_dbi_set:
### push_front cnob
for dbi in new_dbi_set:
LOG.error('AHA detected new integ source: {}'.format(dbi))
### process dbi into branch, branch into more cnobs
### mergesort new cnobs into cnob deque
self.known_dbi_set.update(new_dbi_set)
### return, we'll deal with this later
### +++ save changes and filelog work
# p4 print every revision modified by this changelist.
#
# +++ Also print every revision AFTER this changelist. There's a
# +++ high probability that we'll need those revisons later.
# +++ Printing them all now _greatly_ reduces the total number of
# +++ 'p4 print' requests, reduces the Perforce server's workload
# +++ (and thus repsponse time) in generating incremental file
# +++ revisions from any files stored using RCS deltas (aka most
# +++ files).
with self.p2g.perf.timer[CALC_PRINT]:
depot_path_rev_list = []
for rr in filelog_results:
if ( (not isinstance(rr, dict))
or ('depotFile' not in rr)
or ('rev' not in rr)):
continue
p4file = P4File.create_from_filelog(rr)
p4changelist.files.append(p4file)
depot_path = rr['depotFile']
rev = rr['rev'][0]
if self._already_printed(depot_path, rev):
continue
depot_path_rev_list.append('{}#{},head'
.format(depot_path, rev))
rev_total = len(p4changelist.files)
_debug2('Printing files.'
' change: {change_num}'
' total: {rev_total}'
' need_print: {rev_need_print}'
' already_printed: {rev_already_printed}'
, change_num = cnob.change_num
, rev_need_print = len(depot_path_rev_list)
, rev_already_printed = rev_total - len(depot_path_rev_list)
, rev_total = rev_total)
if depot_path_rev_list:
with self.p2g.perf.timer[PRINT2]:
printhandler = self._print_handler()
server_can_unexpand = self.ctx.p4.server_level > 32
args = ["-a"]
if server_can_unexpand:
args.append("-k")
cmd = ['print'] + args + depot_path_rev_list
with p4gf_util.RawEncoding(self.ctx.p4) \
, p4gf_util.Handler(self.ctx.p4, printhandler) \
, self.ctx.p4.at_exception_level(P4.RAISE_ALL):
self.ctx.p4run(cmd)
printhandler.flush()
# Find each file revision's blob sha1.
for p4file in p4changelist.files:
symlink_path = _depot_rev_to_symlink(
depot_path = p4file.depot_path
, rev = p4file.revision
, symlink_dir = self.symlink_dir )
blob_path = os.readlink(symlink_path)
p4file.sha1 = _blob_path_to_sha1(blob_path)
# If we can copy the Git commit and its tree objects from
# our gitmirror, do so.
# Non-MemCapped code calls all FI functions with
# timer[FAST_IMPORT] as outer container, so must we.
with self.p2g.perf.timer[FAST_IMPORT]:
if self.p2g._fast_import_from_gitmirror(p4changelist, branch):
LOG.debug2('@{} fast-imported from gitmirror.'
.format(cnob.change_num))
return
# Build a git-fast-import commit object.
### _fast_import_from_p4() runs its own filelog to
### discover integ sources. That needs to be hoisted up
### to our own filelog and passed down to avoid
### duplicate work.
LOG.debug2('@{} fast-importing from p4 changelist.'
.format(cnob.change_num))
self.p2g._fast_import_from_p4_mc(
change = p4changelist
, branch = branch
, filelog_results = filelog_results
, mark_to_branch_id = self.mark_to_branch_id
, branch_id_to_temp_name = self.branch_id_to_temp_name )