class P2G:
"""class to manage copying from Perforce to git"""
def __init__(self, ctx):
self.ctx = ctx
self.fastimport = FastImport(self.ctx)
self.fastimport.set_timezone(self.ctx.timezone)
self.fastimport.set_project_root_path(self.ctx.contentlocalroot)
self.perf = p4gf_profiler.TimerCounterSet()
self.perf.add_timers([
OVERALL, (SETUP, OVERALL), (PRINT, OVERALL), (FSTAT, OVERALL),
(SYNC, OVERALL), (FAST_IMPORT, OVERALL), (MIRROR, OVERALL),
(MERGE, OVERALL), (PACK, OVERALL)
])
self.rev_range = None # RevRange instance set in copy().
self.graft_change = None #
self.changes = None # dict['changelist'] ==> P4Changelist of what to copy()
self.printed_revs = None # RevList produced by PrintHandler
self.status_verbose = True
self.progress = ProgressReporter()
def __str__(self):
return "\n".join([
"\n\nFast Import:\n",
str(self.fastimport), "",
str(self.perf), ""
])
def _setup(self, start_at, stop_at):
"""Set RevRange rev_range, figure out which changelists to copy."""
self.rev_range = RevRange.from_start_stop(self.ctx, start_at, stop_at)
LOG.debug(
"Revision range to copy to Git: {rr}".format(rr=self.rev_range))
# get list of changes to import into git
self.changes = P4Changelist.create_changelist_list_as_dict(
self.ctx.p4, self._path_range())
# If grafting, get that too.
if self.rev_range.graft_change_num:
# Ignore all depotFile elements, we just want the change/desc/time/user.
self.graft_change = P4Changelist.create_using_describe(
self.ctx.p4, self.rev_range.graft_change_num,
"ignore_depot_files")
self.graft_change.description += (
'\n[grafted history before {start_at}]'.format(
start_at=start_at))
def _path_range(self):
"""Return the common path...@range string we use frequently.
"""
return self.ctx.client_view_path() + self.rev_range.as_range_string()
def _copy_print(self):
"""p4 print all revs and git-hash-object them into the git repo."""
server_can_unexpand = self.ctx.p4.server_level > 32
printhandler = PrintHandler(need_unexpand=not server_can_unexpand,
tempdir=self.ctx.tempdir.name)
self.ctx.p4.handler = printhandler
args = ["-a"]
if server_can_unexpand:
args.append("-k")
self.ctx.p4.run("print", args, self._path_range())
printhandler.flush()
printhandler.progress.progress_finish()
# If also grafting, print all revs in existence at time of graft.
if self.graft_change:
args = []
if server_can_unexpand:
args.append("-k")
path = self._graft_path()
LOG.debug("Printing for grafted history: {}".format(path))
self.ctx.p4.run("print", args, path)
printhandler.flush()
# If grafting, we just printed revs that refer to changelists
# that have no P4Changelist counterpart in self.changes. Make
# some skeletal versions now so that FstatHandler will have
# someplace to hang its outputStat() P4File instances.
for (_key, p4file) in printhandler.revs.revs:
if not p4file.change in self.changes:
cl = P4Changelist()
cl.change = p4file.change
self.changes[p4file.change] = cl
self.ctx.p4.handler = None
self.printed_revs = printhandler.revs
def _fstat(self):
"""run fstat to find deleted revs and get client paths"""
# TODO for 12.2 print will also report deleted revs so between
# that and using MapApi to get client paths, we won't need this fstat
self.ctx.p4.handler = FstatHandler(self.printed_revs, self.changes)
fstat_cols = "-T" + ",".join(P4File.fstat_cols())
self.ctx.p4.run("fstat", "-Of", fstat_cols, self._path_range())
if self.graft_change:
# Also run 'p4 fstat //<view>/...@change' for the graft
# change to catch all files as of @change, not just
# revs changed between begin and end of _path_range().
self.ctx.p4.run("fstat", fstat_cols, self._graft_path())
self.ctx.p4.handler = None
self._collapse_to_graft_change()
self._add_graft_to_changes()
# don't need this any more
self.printed_revs = None
sorted_changes = [
str(y) for y in sorted([int(x) for x in self.changes.keys()])
]
LOG.debug("\n".join([str(self.changes[ch]) for ch in sorted_changes]))
return sorted_changes
def _sync(self, sorted_changes):
"""fake sync of last change to make life easier at push time"""
self.ctx.p4.handler = SyncHandler()
lastchange = self.changes[sorted_changes[-1]]
self.ctx.p4.run(
"sync", "-kf",
self.ctx.client_view_path() + "@" + str(lastchange.change))
self.ctx.p4.handler = None
def _fast_import(self, sorted_changes, last_commit):
"""build fast-import script from changes, then run fast-import"""
self.progress.progress_init_determinate(len(sorted_changes))
for changenum in sorted_changes:
change = self.changes[changenum]
self.progress.progress_increment("Copying changelists...")
self.ctx.heartbeat()
# create commit and trees
self.fastimport.add_commit(change, last_commit)
last_commit = change.change
# run git-fast-import and get list of marks
marks = self.fastimport.run_fast_import()
# done with these
self.changes = None
return marks
def _mirror(self, marks):
"""build up list of p4 objects to mirror git repo in perforce
then submit them
"""
self.ctx.mirror.add_commits(marks)
self.ctx.mirror.add_objects_to_p4(self.ctx)
LOG.getChild("time").debug("\n\nGit Mirror:\n" + str(self.ctx.mirror))
self.ctx.mirror = GitMirror(self.ctx.config.view_name)
last_commit = marks[len(marks) - 1]
LOG.debug("Last commit created: " + last_commit)
# pylint: disable=R0201
# R0201 Method could be a function
def _pack(self):
"""run 'git gc' to pack up the blobs
aside from any possible performance benefit, this prevents warnings
from git about "unreachable loose objects"
"""
p4gf_util.popen_no_throw(["git", "gc"])
def _collapse_to_graft_change(self):
"""Move all of the files from pre-graft changelists into the graft
changelist. Remove all pre-graft changelists.
NOP if not grafting.
'p4 print //client/...@100' does indeed print all the files that
exist @100, but the tag dict that goes with each file includes the
changelist in which that file was last added/edited, not 100. So
this function gathers up all the file revs with change=1..99 and
sticks them under change 100's file list.
"""
if (not self.graft_change):
return
graft_num_int = int(self.graft_change.change)
LOG.debug("_collapse_to_graft_change() graft_num_int={}".format(
graft_num_int))
# Delete all P4Changelist elements from self.changes where they
# refer to a change that will be collapsed into the graft change,
# including the graft change itself.
del_keys = []
for p4changelist in self.changes.values():
if graft_num_int < int(p4changelist.change):
LOG.debug("_collapse_to_graft_change() skipping {}".format(
p4changelist.change))
continue
LOG.debug("_collapse_to_graft_change() deleting {}".format(
p4changelist.change))
del_keys.append(p4changelist.change)
for key in del_keys:
del self.changes[key]
# Associate with the graft change all printed P4File results from
# graft-change or older
for (_key, p4file) in self.printed_revs.revs:
if graft_num_int < int(p4file.change):
LOG.debug("_collapse_to_graft_change() skipping post-graft {}".
format(p4file))
continue
old = self.graft_change.file_from_depot_path(p4file.depot_path)
# If print picked up multiple revs, keep the newest.
if (not old) or (int(old.change) < int(p4file.change)):
p4file.change = self.graft_change.change
self.graft_change.files.append(p4file)
LOG.debug(
"_collapse_to_graft_change() keeping {}".format(p4file))
else:
LOG.debug(
"_collapse_to_graft_change() skipping, had newer {}".
format(p4file))
def _add_graft_to_changes(self):
"""Add the graft changelist to our list of changes:
It will be copied over like any other change.
NOP if not grafting.
"""
if (not self.graft_change):
return
self.changes[self.graft_change.change] = self.graft_change
def _graft_path(self):
"""If grafting, return '//<client>/...@N' where N is the graft
changelist number.
If not grafting, return None.
"""
if (not self.graft_change):
return
return "{path}@{change}".format(path=self.ctx.client_view_path(),
change=self.graft_change.change)
def copy(self, start_at, stop_at):
"""copy a set of changelists from perforce into git"""
with self.perf.timer[OVERALL]:
with self.perf.timer[SETUP]:
self._setup(start_at, stop_at)
if not len(self.changes):
LOG.debug("No new changes found to copy")
return
last_commit = self.rev_range.last_commit
with self.perf.timer[PRINT]:
self._copy_print()
with self.perf.timer[FSTAT]:
sorted_changes = self._fstat()
with self.perf.timer[SYNC]:
self._sync(sorted_changes)
with self.perf.timer[FAST_IMPORT]:
marks = self._fast_import(sorted_changes, last_commit)
sorted_changes = None
with self.perf.timer[MIRROR]:
self._mirror(marks)
with self.perf.timer[MERGE]:
# merge temporary branch into master, then delete it
self.fastimport.merge()
with self.perf.timer[PACK]:
self._pack()
LOG.getChild("time").debug("\n" + str(self))
class P2G:
"""class to manage copying from Perforce to git"""
def __init__(self, ctx):
self.ctx = ctx
self.fastimport = FastImport(self.ctx)
self.fastimport.set_timezone(self.ctx.timezone)
self.fastimport.set_project_root_path(self.ctx.contentlocalroot)
self.perf = p4gf_profiler.TimerCounterSet()
self.perf.add_timers([OVERALL,
(SETUP, OVERALL),
(PRINT, OVERALL),
(FSTAT, OVERALL),
(SYNC, OVERALL),
(FAST_IMPORT, OVERALL),
(MIRROR, OVERALL),
(MERGE, OVERALL),
(PACK, OVERALL)
])
self.rev_range = None # RevRange instance set in copy().
self.graft_change = None #
self.changes = None # dict['changelist'] ==> P4Changelist of what to copy()
self.printed_revs = None # RevList produced by PrintHandler
self.status_verbose = True
self.progress = ProgressReporter()
def __str__(self):
return "\n".join(["\n\nFast Import:\n",
str(self.fastimport),
"",
str(self.perf),
""
])
def _setup(self, start_at, stop_at):
"""Set RevRange rev_range, figure out which changelists to copy."""
self.rev_range = RevRange.from_start_stop(self.ctx, start_at, stop_at)
LOG.debug("Revision range to copy to Git: {rr}"
.format(rr=self.rev_range))
# get list of changes to import into git
self.changes = P4Changelist.create_changelist_list_as_dict(
self.ctx.p4,
self._path_range())
# If grafting, get that too.
if self.rev_range.graft_change_num:
# Ignore all depotFile elements, we just want the change/desc/time/user.
self.graft_change = P4Changelist.create_using_describe(
self.ctx.p4,
self.rev_range.graft_change_num,
"ignore_depot_files")
self.graft_change.description += ('\n[grafted history before {start_at}]'
.format(start_at=start_at))
def _path_range(self):
"""Return the common path...@range string we use frequently.
"""
return self.ctx.client_view_path() + self.rev_range.as_range_string()
def _copy_print(self):
"""p4 print all revs and git-hash-object them into the git repo."""
server_can_unexpand = self.ctx.p4.server_level > 32
printhandler = PrintHandler(need_unexpand=not server_can_unexpand,
tempdir=self.ctx.tempdir.name)
self.ctx.p4.handler = printhandler
args = ["-a"]
if server_can_unexpand:
args.append("-k")
self.ctx.p4.run("print", args, self._path_range())
printhandler.flush()
printhandler.progress.progress_finish()
# If also grafting, print all revs in existence at time of graft.
if self.graft_change:
args = []
if server_can_unexpand:
args.append("-k")
path = self._graft_path()
LOG.debug("Printing for grafted history: {}".format(path))
self.ctx.p4.run("print", args, path)
printhandler.flush()
# If grafting, we just printed revs that refer to changelists
# that have no P4Changelist counterpart in self.changes. Make
# some skeletal versions now so that FstatHandler will have
# someplace to hang its outputStat() P4File instances.
for (_key, p4file) in printhandler.revs.revs:
if not p4file.change in self.changes:
cl = P4Changelist()
cl.change = p4file.change
self.changes[p4file.change] = cl
self.ctx.p4.handler = None
self.printed_revs = printhandler.revs
def _fstat(self):
"""run fstat to find deleted revs and get client paths"""
# TODO for 12.2 print will also report deleted revs so between
# that and using MapApi to get client paths, we won't need this fstat
self.ctx.p4.handler = FstatHandler(self.printed_revs, self.changes)
fstat_cols = "-T" + ",".join(P4File.fstat_cols())
self.ctx.p4.run("fstat", "-Of", fstat_cols, self._path_range())
if self.graft_change:
# Also run 'p4 fstat //<view>/...@change' for the graft
# change to catch all files as of @change, not just
# revs changed between begin and end of _path_range().
self.ctx.p4.run("fstat", fstat_cols, self._graft_path())
self.ctx.p4.handler = None
self._collapse_to_graft_change()
self._add_graft_to_changes()
# don't need this any more
self.printed_revs = None
sorted_changes = [str(y) for y in sorted([int(x) for x in self.changes.keys()])]
LOG.debug("\n".join([str(self.changes[ch]) for ch in sorted_changes]))
return sorted_changes
def _sync(self, sorted_changes):
"""fake sync of last change to make life easier at push time"""
self.ctx.p4.handler = SyncHandler()
lastchange = self.changes[sorted_changes[-1]]
self.ctx.p4.run("sync", "-kf",
self.ctx.client_view_path() + "@" + str(lastchange.change))
self.ctx.p4.handler = None
def _fast_import(self, sorted_changes, last_commit):
"""build fast-import script from changes, then run fast-import"""
self.progress.progress_init_determinate(len(sorted_changes))
for changenum in sorted_changes:
change = self.changes[changenum]
self.progress.progress_increment("Copying changelists...")
self.ctx.heartbeat()
# create commit and trees
self.fastimport.add_commit(change, last_commit)
last_commit = change.change
# run git-fast-import and get list of marks
marks = self.fastimport.run_fast_import()
# done with these
self.changes = None
return marks
def _mirror(self, marks):
"""build up list of p4 objects to mirror git repo in perforce
then submit them
"""
self.ctx.mirror.add_commits(marks)
self.ctx.mirror.add_objects_to_p4(self.ctx)
LOG.getChild("time").debug("\n\nGit Mirror:\n" + str(self.ctx.mirror))
self.ctx.mirror = GitMirror(self.ctx.config.view_name)
last_commit = marks[len(marks) - 1]
LOG.debug("Last commit created: " + last_commit)
# pylint: disable=R0201
# R0201 Method could be a function
def _pack(self):
"""run 'git gc' to pack up the blobs
aside from any possible performance benefit, this prevents warnings
from git about "unreachable loose objects"
"""
p4gf_util.popen_no_throw(["git", "gc"])
def _collapse_to_graft_change(self):
"""Move all of the files from pre-graft changelists into the graft
changelist. Remove all pre-graft changelists.
NOP if not grafting.
'p4 print //client/...@100' does indeed print all the files that
exist @100, but the tag dict that goes with each file includes the
changelist in which that file was last added/edited, not 100. So
this function gathers up all the file revs with change=1..99 and
sticks them under change 100's file list.
"""
if (not self.graft_change):
return
graft_num_int = int(self.graft_change.change)
LOG.debug("_collapse_to_graft_change() graft_num_int={}".format(graft_num_int))
# Delete all P4Changelist elements from self.changes where they
# refer to a change that will be collapsed into the graft change,
# including the graft change itself.
del_keys = []
for p4changelist in self.changes.values():
if graft_num_int < int(p4changelist.change):
LOG.debug("_collapse_to_graft_change() skipping {}".format(p4changelist.change))
continue
LOG.debug("_collapse_to_graft_change() deleting {}".format(p4changelist.change))
del_keys.append(p4changelist.change)
for key in del_keys:
del self.changes[key]
# Associate with the graft change all printed P4File results from
# graft-change or older
for (_key, p4file) in self.printed_revs.revs:
if graft_num_int < int(p4file.change):
LOG.debug("_collapse_to_graft_change() skipping post-graft {}".format(p4file))
continue
old = self.graft_change.file_from_depot_path(p4file.depot_path)
# If print picked up multiple revs, keep the newest.
if (not old) or (int(old.change) < int(p4file.change)):
p4file.change = self.graft_change.change
self.graft_change.files.append(p4file)
LOG.debug("_collapse_to_graft_change() keeping {}".format(p4file))
else:
LOG.debug("_collapse_to_graft_change() skipping, had newer {}".format(p4file))
def _add_graft_to_changes(self):
"""Add the graft changelist to our list of changes:
It will be copied over like any other change.
NOP if not grafting.
"""
if (not self.graft_change):
return
self.changes[self.graft_change.change] = self.graft_change
def _graft_path(self):
"""If grafting, return '//<client>/...@N' where N is the graft
changelist number.
If not grafting, return None.
"""
if (not self.graft_change):
return
return "{path}@{change}".format(
path = self.ctx.client_view_path(),
change = self.graft_change.change)
def copy(self, start_at, stop_at):
"""copy a set of changelists from perforce into git"""
with self.perf.timer[OVERALL]:
with self.perf.timer[SETUP]:
self._setup(start_at, stop_at)
if not len(self.changes):
LOG.debug("No new changes found to copy")
return
last_commit = self.rev_range.last_commit
with self.perf.timer[PRINT]:
self._copy_print()
with self.perf.timer[FSTAT]:
sorted_changes = self._fstat()
with self.perf.timer[SYNC]:
self._sync(sorted_changes)
with self.perf.timer[FAST_IMPORT]:
marks = self._fast_import(sorted_changes, last_commit)
sorted_changes = None
with self.perf.timer[MIRROR]:
self._mirror(marks)
with self.perf.timer[MERGE]:
# merge temporary branch into master, then delete it
self.fastimport.merge()
with self.perf.timer[PACK]:
self._pack()
LOG.getChild("time").debug("\n" + str(self))