def __init__(self, ctx): self.ctx = ctx self.fastimport = FastImport(self.ctx) self.fastimport.set_timezone(self.ctx.timezone) self.fastimport.set_project_root_path(self.ctx.contentlocalroot) self.perf = p4gf_profiler.TimerCounterSet() self.perf.add_timers([ OVERALL, (SETUP, OVERALL), (PRINT, OVERALL), (FSTAT, OVERALL), (SYNC, OVERALL), (FAST_IMPORT, OVERALL), (MIRROR, OVERALL), (MERGE, OVERALL), (PACK, OVERALL) ]) self.rev_range = None # RevRange instance set in copy(). self.graft_change = None # self.changes = None # dict['changelist'] ==> P4Changelist of what to copy() self.printed_revs = None # RevList produced by PrintHandler self.status_verbose = True self.progress = ProgressReporter()
def __init__(self, ctx): self.ctx = ctx self.fastimport = FastImport(self.ctx) self.fastimport.set_timezone(self.ctx.timezone) self.fastimport.set_project_root_path(self.ctx.contentlocalroot) self.perf = p4gf_profiler.TimerCounterSet() self.perf.add_timers([OVERALL, (SETUP, OVERALL), (PRINT, OVERALL), (FSTAT, OVERALL), (SYNC, OVERALL), (FAST_IMPORT, OVERALL), (MIRROR, OVERALL), (MERGE, OVERALL), (PACK, OVERALL) ]) self.rev_range = None # RevRange instance set in copy(). self.graft_change = None # self.changes = None # dict['changelist'] ==> P4Changelist of what to copy() self.printed_revs = None # RevList produced by PrintHandler self.status_verbose = True self.progress = ProgressReporter()
class P2G: """class to manage copying from Perforce to git""" def __init__(self, ctx): self.ctx = ctx self.fastimport = FastImport(self.ctx) self.fastimport.set_timezone(self.ctx.timezone) self.fastimport.set_project_root_path(self.ctx.contentlocalroot) self.perf = p4gf_profiler.TimerCounterSet() self.perf.add_timers([ OVERALL, (SETUP, OVERALL), (PRINT, OVERALL), (FSTAT, OVERALL), (SYNC, OVERALL), (FAST_IMPORT, OVERALL), (MIRROR, OVERALL), (MERGE, OVERALL), (PACK, OVERALL) ]) self.rev_range = None # RevRange instance set in copy(). self.graft_change = None # self.changes = None # dict['changelist'] ==> P4Changelist of what to copy() self.printed_revs = None # RevList produced by PrintHandler self.status_verbose = True self.progress = ProgressReporter() def __str__(self): return "\n".join([ "\n\nFast Import:\n", str(self.fastimport), "", str(self.perf), "" ]) def _setup(self, start_at, stop_at): """Set RevRange rev_range, figure out which changelists to copy.""" self.rev_range = RevRange.from_start_stop(self.ctx, start_at, stop_at) LOG.debug( "Revision range to copy to Git: {rr}".format(rr=self.rev_range)) # get list of changes to import into git self.changes = P4Changelist.create_changelist_list_as_dict( self.ctx.p4, self._path_range()) # If grafting, get that too. if self.rev_range.graft_change_num: # Ignore all depotFile elements, we just want the change/desc/time/user. self.graft_change = P4Changelist.create_using_describe( self.ctx.p4, self.rev_range.graft_change_num, "ignore_depot_files") self.graft_change.description += ( '\n[grafted history before {start_at}]'.format( start_at=start_at)) def _path_range(self): """Return the common path...@range string we use frequently. """ return self.ctx.client_view_path() + self.rev_range.as_range_string() def _copy_print(self): """p4 print all revs and git-hash-object them into the git repo.""" server_can_unexpand = self.ctx.p4.server_level > 32 printhandler = PrintHandler(need_unexpand=not server_can_unexpand, tempdir=self.ctx.tempdir.name) self.ctx.p4.handler = printhandler args = ["-a"] if server_can_unexpand: args.append("-k") self.ctx.p4.run("print", args, self._path_range()) printhandler.flush() printhandler.progress.progress_finish() # If also grafting, print all revs in existence at time of graft. if self.graft_change: args = [] if server_can_unexpand: args.append("-k") path = self._graft_path() LOG.debug("Printing for grafted history: {}".format(path)) self.ctx.p4.run("print", args, path) printhandler.flush() # If grafting, we just printed revs that refer to changelists # that have no P4Changelist counterpart in self.changes. Make # some skeletal versions now so that FstatHandler will have # someplace to hang its outputStat() P4File instances. for (_key, p4file) in printhandler.revs.revs: if not p4file.change in self.changes: cl = P4Changelist() cl.change = p4file.change self.changes[p4file.change] = cl self.ctx.p4.handler = None self.printed_revs = printhandler.revs def _fstat(self): """run fstat to find deleted revs and get client paths""" # TODO for 12.2 print will also report deleted revs so between # that and using MapApi to get client paths, we won't need this fstat self.ctx.p4.handler = FstatHandler(self.printed_revs, self.changes) fstat_cols = "-T" + ",".join(P4File.fstat_cols()) self.ctx.p4.run("fstat", "-Of", fstat_cols, self._path_range()) if self.graft_change: # Also run 'p4 fstat //<view>/...@change' for the graft # change to catch all files as of @change, not just # revs changed between begin and end of _path_range(). self.ctx.p4.run("fstat", fstat_cols, self._graft_path()) self.ctx.p4.handler = None self._collapse_to_graft_change() self._add_graft_to_changes() # don't need this any more self.printed_revs = None sorted_changes = [ str(y) for y in sorted([int(x) for x in self.changes.keys()]) ] LOG.debug("\n".join([str(self.changes[ch]) for ch in sorted_changes])) return sorted_changes def _sync(self, sorted_changes): """fake sync of last change to make life easier at push time""" self.ctx.p4.handler = SyncHandler() lastchange = self.changes[sorted_changes[-1]] self.ctx.p4.run( "sync", "-kf", self.ctx.client_view_path() + "@" + str(lastchange.change)) self.ctx.p4.handler = None def _fast_import(self, sorted_changes, last_commit): """build fast-import script from changes, then run fast-import""" self.progress.progress_init_determinate(len(sorted_changes)) for changenum in sorted_changes: change = self.changes[changenum] self.progress.progress_increment("Copying changelists...") self.ctx.heartbeat() # create commit and trees self.fastimport.add_commit(change, last_commit) last_commit = change.change # run git-fast-import and get list of marks marks = self.fastimport.run_fast_import() # done with these self.changes = None return marks def _mirror(self, marks): """build up list of p4 objects to mirror git repo in perforce then submit them """ self.ctx.mirror.add_commits(marks) self.ctx.mirror.add_objects_to_p4(self.ctx) LOG.getChild("time").debug("\n\nGit Mirror:\n" + str(self.ctx.mirror)) self.ctx.mirror = GitMirror(self.ctx.config.view_name) last_commit = marks[len(marks) - 1] LOG.debug("Last commit created: " + last_commit) # pylint: disable=R0201 # R0201 Method could be a function def _pack(self): """run 'git gc' to pack up the blobs aside from any possible performance benefit, this prevents warnings from git about "unreachable loose objects" """ p4gf_util.popen_no_throw(["git", "gc"]) def _collapse_to_graft_change(self): """Move all of the files from pre-graft changelists into the graft changelist. Remove all pre-graft changelists. NOP if not grafting. 'p4 print //client/...@100' does indeed print all the files that exist @100, but the tag dict that goes with each file includes the changelist in which that file was last added/edited, not 100. So this function gathers up all the file revs with change=1..99 and sticks them under change 100's file list. """ if (not self.graft_change): return graft_num_int = int(self.graft_change.change) LOG.debug("_collapse_to_graft_change() graft_num_int={}".format( graft_num_int)) # Delete all P4Changelist elements from self.changes where they # refer to a change that will be collapsed into the graft change, # including the graft change itself. del_keys = [] for p4changelist in self.changes.values(): if graft_num_int < int(p4changelist.change): LOG.debug("_collapse_to_graft_change() skipping {}".format( p4changelist.change)) continue LOG.debug("_collapse_to_graft_change() deleting {}".format( p4changelist.change)) del_keys.append(p4changelist.change) for key in del_keys: del self.changes[key] # Associate with the graft change all printed P4File results from # graft-change or older for (_key, p4file) in self.printed_revs.revs: if graft_num_int < int(p4file.change): LOG.debug("_collapse_to_graft_change() skipping post-graft {}". format(p4file)) continue old = self.graft_change.file_from_depot_path(p4file.depot_path) # If print picked up multiple revs, keep the newest. if (not old) or (int(old.change) < int(p4file.change)): p4file.change = self.graft_change.change self.graft_change.files.append(p4file) LOG.debug( "_collapse_to_graft_change() keeping {}".format(p4file)) else: LOG.debug( "_collapse_to_graft_change() skipping, had newer {}". format(p4file)) def _add_graft_to_changes(self): """Add the graft changelist to our list of changes: It will be copied over like any other change. NOP if not grafting. """ if (not self.graft_change): return self.changes[self.graft_change.change] = self.graft_change def _graft_path(self): """If grafting, return '//<client>/...@N' where N is the graft changelist number. If not grafting, return None. """ if (not self.graft_change): return return "{path}@{change}".format(path=self.ctx.client_view_path(), change=self.graft_change.change) def copy(self, start_at, stop_at): """copy a set of changelists from perforce into git""" with self.perf.timer[OVERALL]: with self.perf.timer[SETUP]: self._setup(start_at, stop_at) if not len(self.changes): LOG.debug("No new changes found to copy") return last_commit = self.rev_range.last_commit with self.perf.timer[PRINT]: self._copy_print() with self.perf.timer[FSTAT]: sorted_changes = self._fstat() with self.perf.timer[SYNC]: self._sync(sorted_changes) with self.perf.timer[FAST_IMPORT]: marks = self._fast_import(sorted_changes, last_commit) sorted_changes = None with self.perf.timer[MIRROR]: self._mirror(marks) with self.perf.timer[MERGE]: # merge temporary branch into master, then delete it self.fastimport.merge() with self.perf.timer[PACK]: self._pack() LOG.getChild("time").debug("\n" + str(self))
class P2G: """class to manage copying from Perforce to git""" def __init__(self, ctx): self.ctx = ctx self.fastimport = FastImport(self.ctx) self.fastimport.set_timezone(self.ctx.timezone) self.fastimport.set_project_root_path(self.ctx.contentlocalroot) self.perf = p4gf_profiler.TimerCounterSet() self.perf.add_timers([OVERALL, (SETUP, OVERALL), (PRINT, OVERALL), (FSTAT, OVERALL), (SYNC, OVERALL), (FAST_IMPORT, OVERALL), (MIRROR, OVERALL), (MERGE, OVERALL), (PACK, OVERALL) ]) self.rev_range = None # RevRange instance set in copy(). self.graft_change = None # self.changes = None # dict['changelist'] ==> P4Changelist of what to copy() self.printed_revs = None # RevList produced by PrintHandler self.status_verbose = True self.progress = ProgressReporter() def __str__(self): return "\n".join(["\n\nFast Import:\n", str(self.fastimport), "", str(self.perf), "" ]) def _setup(self, start_at, stop_at): """Set RevRange rev_range, figure out which changelists to copy.""" self.rev_range = RevRange.from_start_stop(self.ctx, start_at, stop_at) LOG.debug("Revision range to copy to Git: {rr}" .format(rr=self.rev_range)) # get list of changes to import into git self.changes = P4Changelist.create_changelist_list_as_dict( self.ctx.p4, self._path_range()) # If grafting, get that too. if self.rev_range.graft_change_num: # Ignore all depotFile elements, we just want the change/desc/time/user. self.graft_change = P4Changelist.create_using_describe( self.ctx.p4, self.rev_range.graft_change_num, "ignore_depot_files") self.graft_change.description += ('\n[grafted history before {start_at}]' .format(start_at=start_at)) def _path_range(self): """Return the common path...@range string we use frequently. """ return self.ctx.client_view_path() + self.rev_range.as_range_string() def _copy_print(self): """p4 print all revs and git-hash-object them into the git repo.""" server_can_unexpand = self.ctx.p4.server_level > 32 printhandler = PrintHandler(need_unexpand=not server_can_unexpand, tempdir=self.ctx.tempdir.name) self.ctx.p4.handler = printhandler args = ["-a"] if server_can_unexpand: args.append("-k") self.ctx.p4.run("print", args, self._path_range()) printhandler.flush() printhandler.progress.progress_finish() # If also grafting, print all revs in existence at time of graft. if self.graft_change: args = [] if server_can_unexpand: args.append("-k") path = self._graft_path() LOG.debug("Printing for grafted history: {}".format(path)) self.ctx.p4.run("print", args, path) printhandler.flush() # If grafting, we just printed revs that refer to changelists # that have no P4Changelist counterpart in self.changes. Make # some skeletal versions now so that FstatHandler will have # someplace to hang its outputStat() P4File instances. for (_key, p4file) in printhandler.revs.revs: if not p4file.change in self.changes: cl = P4Changelist() cl.change = p4file.change self.changes[p4file.change] = cl self.ctx.p4.handler = None self.printed_revs = printhandler.revs def _fstat(self): """run fstat to find deleted revs and get client paths""" # TODO for 12.2 print will also report deleted revs so between # that and using MapApi to get client paths, we won't need this fstat self.ctx.p4.handler = FstatHandler(self.printed_revs, self.changes) fstat_cols = "-T" + ",".join(P4File.fstat_cols()) self.ctx.p4.run("fstat", "-Of", fstat_cols, self._path_range()) if self.graft_change: # Also run 'p4 fstat //<view>/...@change' for the graft # change to catch all files as of @change, not just # revs changed between begin and end of _path_range(). self.ctx.p4.run("fstat", fstat_cols, self._graft_path()) self.ctx.p4.handler = None self._collapse_to_graft_change() self._add_graft_to_changes() # don't need this any more self.printed_revs = None sorted_changes = [str(y) for y in sorted([int(x) for x in self.changes.keys()])] LOG.debug("\n".join([str(self.changes[ch]) for ch in sorted_changes])) return sorted_changes def _sync(self, sorted_changes): """fake sync of last change to make life easier at push time""" self.ctx.p4.handler = SyncHandler() lastchange = self.changes[sorted_changes[-1]] self.ctx.p4.run("sync", "-kf", self.ctx.client_view_path() + "@" + str(lastchange.change)) self.ctx.p4.handler = None def _fast_import(self, sorted_changes, last_commit): """build fast-import script from changes, then run fast-import""" self.progress.progress_init_determinate(len(sorted_changes)) for changenum in sorted_changes: change = self.changes[changenum] self.progress.progress_increment("Copying changelists...") self.ctx.heartbeat() # create commit and trees self.fastimport.add_commit(change, last_commit) last_commit = change.change # run git-fast-import and get list of marks marks = self.fastimport.run_fast_import() # done with these self.changes = None return marks def _mirror(self, marks): """build up list of p4 objects to mirror git repo in perforce then submit them """ self.ctx.mirror.add_commits(marks) self.ctx.mirror.add_objects_to_p4(self.ctx) LOG.getChild("time").debug("\n\nGit Mirror:\n" + str(self.ctx.mirror)) self.ctx.mirror = GitMirror(self.ctx.config.view_name) last_commit = marks[len(marks) - 1] LOG.debug("Last commit created: " + last_commit) # pylint: disable=R0201 # R0201 Method could be a function def _pack(self): """run 'git gc' to pack up the blobs aside from any possible performance benefit, this prevents warnings from git about "unreachable loose objects" """ p4gf_util.popen_no_throw(["git", "gc"]) def _collapse_to_graft_change(self): """Move all of the files from pre-graft changelists into the graft changelist. Remove all pre-graft changelists. NOP if not grafting. 'p4 print //client/...@100' does indeed print all the files that exist @100, but the tag dict that goes with each file includes the changelist in which that file was last added/edited, not 100. So this function gathers up all the file revs with change=1..99 and sticks them under change 100's file list. """ if (not self.graft_change): return graft_num_int = int(self.graft_change.change) LOG.debug("_collapse_to_graft_change() graft_num_int={}".format(graft_num_int)) # Delete all P4Changelist elements from self.changes where they # refer to a change that will be collapsed into the graft change, # including the graft change itself. del_keys = [] for p4changelist in self.changes.values(): if graft_num_int < int(p4changelist.change): LOG.debug("_collapse_to_graft_change() skipping {}".format(p4changelist.change)) continue LOG.debug("_collapse_to_graft_change() deleting {}".format(p4changelist.change)) del_keys.append(p4changelist.change) for key in del_keys: del self.changes[key] # Associate with the graft change all printed P4File results from # graft-change or older for (_key, p4file) in self.printed_revs.revs: if graft_num_int < int(p4file.change): LOG.debug("_collapse_to_graft_change() skipping post-graft {}".format(p4file)) continue old = self.graft_change.file_from_depot_path(p4file.depot_path) # If print picked up multiple revs, keep the newest. if (not old) or (int(old.change) < int(p4file.change)): p4file.change = self.graft_change.change self.graft_change.files.append(p4file) LOG.debug("_collapse_to_graft_change() keeping {}".format(p4file)) else: LOG.debug("_collapse_to_graft_change() skipping, had newer {}".format(p4file)) def _add_graft_to_changes(self): """Add the graft changelist to our list of changes: It will be copied over like any other change. NOP if not grafting. """ if (not self.graft_change): return self.changes[self.graft_change.change] = self.graft_change def _graft_path(self): """If grafting, return '//<client>/...@N' where N is the graft changelist number. If not grafting, return None. """ if (not self.graft_change): return return "{path}@{change}".format( path = self.ctx.client_view_path(), change = self.graft_change.change) def copy(self, start_at, stop_at): """copy a set of changelists from perforce into git""" with self.perf.timer[OVERALL]: with self.perf.timer[SETUP]: self._setup(start_at, stop_at) if not len(self.changes): LOG.debug("No new changes found to copy") return last_commit = self.rev_range.last_commit with self.perf.timer[PRINT]: self._copy_print() with self.perf.timer[FSTAT]: sorted_changes = self._fstat() with self.perf.timer[SYNC]: self._sync(sorted_changes) with self.perf.timer[FAST_IMPORT]: marks = self._fast_import(sorted_changes, last_commit) sorted_changes = None with self.perf.timer[MIRROR]: self._mirror(marks) with self.perf.timer[MERGE]: # merge temporary branch into master, then delete it self.fastimport.merge() with self.perf.timer[PACK]: self._pack() LOG.getChild("time").debug("\n" + str(self))