Beispiel #1
0
class P2G:
    """class to manage copying from Perforce to git"""
    def __init__(self, ctx):
        self.ctx = ctx
        self.fastimport = FastImport(self.ctx)
        self.fastimport.set_timezone(self.ctx.timezone)
        self.fastimport.set_project_root_path(self.ctx.contentlocalroot)
        self.perf = p4gf_profiler.TimerCounterSet()
        self.perf.add_timers([
            OVERALL, (SETUP, OVERALL), (PRINT, OVERALL), (FSTAT, OVERALL),
            (SYNC, OVERALL), (FAST_IMPORT, OVERALL), (MIRROR, OVERALL),
            (MERGE, OVERALL), (PACK, OVERALL)
        ])

        self.rev_range = None  # RevRange instance set in copy().
        self.graft_change = None  #
        self.changes = None  # dict['changelist'] ==> P4Changelist of what to copy()
        self.printed_revs = None  # RevList produced by PrintHandler
        self.status_verbose = True
        self.progress = ProgressReporter()

    def __str__(self):
        return "\n".join([
            "\n\nFast Import:\n",
            str(self.fastimport), "",
            str(self.perf), ""
        ])

    def _setup(self, start_at, stop_at):
        """Set RevRange rev_range, figure out which changelists to copy."""
        self.rev_range = RevRange.from_start_stop(self.ctx, start_at, stop_at)
        LOG.debug(
            "Revision range to copy to Git: {rr}".format(rr=self.rev_range))

        # get list of changes to import into git
        self.changes = P4Changelist.create_changelist_list_as_dict(
            self.ctx.p4, self._path_range())

        # If grafting, get that too.
        if self.rev_range.graft_change_num:
            # Ignore all depotFile elements, we just want the change/desc/time/user.
            self.graft_change = P4Changelist.create_using_describe(
                self.ctx.p4, self.rev_range.graft_change_num,
                "ignore_depot_files")
            self.graft_change.description += (
                '\n[grafted history before {start_at}]'.format(
                    start_at=start_at))

    def _path_range(self):
        """Return the common path...@range string we use frequently.
        """
        return self.ctx.client_view_path() + self.rev_range.as_range_string()

    def _copy_print(self):
        """p4 print all revs and git-hash-object them into the git repo."""
        server_can_unexpand = self.ctx.p4.server_level > 32
        printhandler = PrintHandler(need_unexpand=not server_can_unexpand,
                                    tempdir=self.ctx.tempdir.name)
        self.ctx.p4.handler = printhandler
        args = ["-a"]
        if server_can_unexpand:
            args.append("-k")
        self.ctx.p4.run("print", args, self._path_range())
        printhandler.flush()
        printhandler.progress.progress_finish()

        # If also grafting, print all revs in existence at time of graft.
        if self.graft_change:
            args = []
            if server_can_unexpand:
                args.append("-k")
            path = self._graft_path()
            LOG.debug("Printing for grafted history: {}".format(path))
            self.ctx.p4.run("print", args, path)
            printhandler.flush()

            # If grafting, we just printed revs that refer to changelists
            # that have no P4Changelist counterpart in self.changes. Make
            # some skeletal versions now so that FstatHandler will have
            # someplace to hang its outputStat() P4File instances.
            for (_key, p4file) in printhandler.revs.revs:
                if not p4file.change in self.changes:
                    cl = P4Changelist()
                    cl.change = p4file.change
                    self.changes[p4file.change] = cl

        self.ctx.p4.handler = None
        self.printed_revs = printhandler.revs

    def _fstat(self):
        """run fstat to find deleted revs and get client paths"""
        # TODO for 12.2 print will also report deleted revs so between
        # that and using MapApi to get client paths, we won't need this fstat
        self.ctx.p4.handler = FstatHandler(self.printed_revs, self.changes)
        fstat_cols = "-T" + ",".join(P4File.fstat_cols())
        self.ctx.p4.run("fstat", "-Of", fstat_cols, self._path_range())

        if self.graft_change:
            # Also run 'p4 fstat //<view>/...@change' for the graft
            # change to catch all files as of @change, not just
            # revs changed between begin and end of _path_range().
            self.ctx.p4.run("fstat", fstat_cols, self._graft_path())

        self.ctx.p4.handler = None

        self._collapse_to_graft_change()
        self._add_graft_to_changes()

        # don't need this any more
        self.printed_revs = None

        sorted_changes = [
            str(y) for y in sorted([int(x) for x in self.changes.keys()])
        ]

        LOG.debug("\n".join([str(self.changes[ch]) for ch in sorted_changes]))
        return sorted_changes

    def _sync(self, sorted_changes):
        """fake sync of last change to make life easier at push time"""
        self.ctx.p4.handler = SyncHandler()
        lastchange = self.changes[sorted_changes[-1]]
        self.ctx.p4.run(
            "sync", "-kf",
            self.ctx.client_view_path() + "@" + str(lastchange.change))
        self.ctx.p4.handler = None

    def _fast_import(self, sorted_changes, last_commit):
        """build fast-import script from changes, then run fast-import"""
        self.progress.progress_init_determinate(len(sorted_changes))
        for changenum in sorted_changes:
            change = self.changes[changenum]
            self.progress.progress_increment("Copying changelists...")
            self.ctx.heartbeat()

            # create commit and trees
            self.fastimport.add_commit(change, last_commit)

            last_commit = change.change

        # run git-fast-import and get list of marks
        marks = self.fastimport.run_fast_import()

        # done with these
        self.changes = None
        return marks

    def _mirror(self, marks):
        """build up list of p4 objects to mirror git repo in perforce
        then submit them
        """
        self.ctx.mirror.add_commits(marks)
        self.ctx.mirror.add_objects_to_p4(self.ctx)
        LOG.getChild("time").debug("\n\nGit Mirror:\n" + str(self.ctx.mirror))
        self.ctx.mirror = GitMirror(self.ctx.config.view_name)

        last_commit = marks[len(marks) - 1]
        LOG.debug("Last commit created: " + last_commit)

    # pylint: disable=R0201
    # R0201 Method could be a function
    def _pack(self):
        """run 'git gc' to pack up the blobs

        aside from any possible performance benefit, this prevents warnings
        from git about "unreachable loose objects"
        """
        p4gf_util.popen_no_throw(["git", "gc"])

    def _collapse_to_graft_change(self):
        """Move all of the files from pre-graft changelists into the graft
        changelist. Remove all pre-graft changelists.

        NOP if not grafting.

        'p4 print //client/...@100' does indeed print all the files that
        exist @100, but the tag dict that goes with each file includes the
        changelist in which that file was last added/edited, not 100. So
        this function gathers up all the file revs with change=1..99 and
        sticks them under change 100's file list.
        """
        if (not self.graft_change):
            return
        graft_num_int = int(self.graft_change.change)
        LOG.debug("_collapse_to_graft_change() graft_num_int={}".format(
            graft_num_int))

        # Delete all P4Changelist elements from self.changes where they
        # refer to a change that will be collapsed into the graft change,
        # including the graft change itself.
        del_keys = []
        for p4changelist in self.changes.values():
            if graft_num_int < int(p4changelist.change):
                LOG.debug("_collapse_to_graft_change() skipping {}".format(
                    p4changelist.change))
                continue

            LOG.debug("_collapse_to_graft_change() deleting {}".format(
                p4changelist.change))
            del_keys.append(p4changelist.change)
        for key in del_keys:
            del self.changes[key]

        # Associate with the graft change all printed P4File results from
        # graft-change or older
        for (_key, p4file) in self.printed_revs.revs:
            if graft_num_int < int(p4file.change):
                LOG.debug("_collapse_to_graft_change() skipping post-graft {}".
                          format(p4file))
                continue

            old = self.graft_change.file_from_depot_path(p4file.depot_path)
            # If print picked up multiple revs, keep the newest.
            if (not old) or (int(old.change) < int(p4file.change)):
                p4file.change = self.graft_change.change
                self.graft_change.files.append(p4file)
                LOG.debug(
                    "_collapse_to_graft_change() keeping {}".format(p4file))
            else:
                LOG.debug(
                    "_collapse_to_graft_change() skipping, had newer  {}".
                    format(p4file))

    def _add_graft_to_changes(self):
        """Add the graft changelist to our list of changes:
        It will be copied over like any other change.

        NOP if not grafting.
        """
        if (not self.graft_change):
            return
        self.changes[self.graft_change.change] = self.graft_change

    def _graft_path(self):
        """If grafting, return '//<client>/...@N' where N is the graft
        changelist number.

        If not grafting, return None.
        """
        if (not self.graft_change):
            return
        return "{path}@{change}".format(path=self.ctx.client_view_path(),
                                        change=self.graft_change.change)

    def copy(self, start_at, stop_at):
        """copy a set of changelists from perforce into git"""

        with self.perf.timer[OVERALL]:
            with self.perf.timer[SETUP]:
                self._setup(start_at, stop_at)

                if not len(self.changes):
                    LOG.debug("No new changes found to copy")
                    return

                last_commit = self.rev_range.last_commit

            with self.perf.timer[PRINT]:
                self._copy_print()

            with self.perf.timer[FSTAT]:
                sorted_changes = self._fstat()

            with self.perf.timer[SYNC]:
                self._sync(sorted_changes)

            with self.perf.timer[FAST_IMPORT]:
                marks = self._fast_import(sorted_changes, last_commit)
                sorted_changes = None

            with self.perf.timer[MIRROR]:
                self._mirror(marks)

            with self.perf.timer[MERGE]:
                # merge temporary branch into master, then delete it
                self.fastimport.merge()

            with self.perf.timer[PACK]:
                self._pack()

        LOG.getChild("time").debug("\n" + str(self))
class P2G:
    """class to manage copying from Perforce to git"""
    def __init__(self, ctx):
        self.ctx = ctx
        self.fastimport = FastImport(self.ctx)
        self.fastimport.set_timezone(self.ctx.timezone)
        self.fastimport.set_project_root_path(self.ctx.contentlocalroot)
        self.perf = p4gf_profiler.TimerCounterSet()
        self.perf.add_timers([OVERALL,
                            (SETUP, OVERALL),
                            (PRINT, OVERALL),
                            (FSTAT, OVERALL),
                            (SYNC, OVERALL),
                            (FAST_IMPORT, OVERALL),
                            (MIRROR, OVERALL),
                            (MERGE, OVERALL),
                            (PACK, OVERALL)
                            ])

        self.rev_range      = None  # RevRange instance set in copy().
        self.graft_change   = None  #
        self.changes        = None  # dict['changelist'] ==> P4Changelist of what to copy()
        self.printed_revs   = None  # RevList produced by PrintHandler
        self.status_verbose = True
        self.progress       = ProgressReporter()

    def __str__(self):
        return "\n".join(["\n\nFast Import:\n",
                          str(self.fastimport),
                          "",
                          str(self.perf),
                          ""
                          ])

    def _setup(self, start_at, stop_at):
        """Set RevRange rev_range, figure out which changelists to copy."""
        self.rev_range = RevRange.from_start_stop(self.ctx, start_at, stop_at)
        LOG.debug("Revision range to copy to Git: {rr}"
                  .format(rr=self.rev_range))

        # get list of changes to import into git
        self.changes = P4Changelist.create_changelist_list_as_dict(
                            self.ctx.p4,
                            self._path_range())

        # If grafting, get that too.
        if self.rev_range.graft_change_num:
            # Ignore all depotFile elements, we just want the change/desc/time/user.
            self.graft_change = P4Changelist.create_using_describe(
                                    self.ctx.p4,
                                    self.rev_range.graft_change_num,
                                    "ignore_depot_files")
            self.graft_change.description += ('\n[grafted history before {start_at}]'
                                              .format(start_at=start_at))

    def _path_range(self):
        """Return the common path...@range string we use frequently.
        """
        return self.ctx.client_view_path() + self.rev_range.as_range_string()

    def _copy_print(self):
        """p4 print all revs and git-hash-object them into the git repo."""
        server_can_unexpand = self.ctx.p4.server_level > 32
        printhandler = PrintHandler(need_unexpand=not server_can_unexpand,
                                    tempdir=self.ctx.tempdir.name)
        self.ctx.p4.handler = printhandler
        args = ["-a"]
        if server_can_unexpand:
            args.append("-k")
        self.ctx.p4.run("print", args, self._path_range())
        printhandler.flush()
        printhandler.progress.progress_finish()

        # If also grafting, print all revs in existence at time of graft.
        if self.graft_change:
            args = []
            if server_can_unexpand:
                args.append("-k")
            path = self._graft_path()
            LOG.debug("Printing for grafted history: {}".format(path))
            self.ctx.p4.run("print", args, path)
            printhandler.flush()

            # If grafting, we just printed revs that refer to changelists
            # that have no P4Changelist counterpart in self.changes. Make
            # some skeletal versions now so that FstatHandler will have
            # someplace to hang its outputStat() P4File instances.
            for (_key, p4file) in printhandler.revs.revs:
                if not p4file.change in self.changes:
                    cl = P4Changelist()
                    cl.change = p4file.change
                    self.changes[p4file.change] = cl

        self.ctx.p4.handler = None
        self.printed_revs = printhandler.revs

    def _fstat(self):
        """run fstat to find deleted revs and get client paths"""
        # TODO for 12.2 print will also report deleted revs so between
        # that and using MapApi to get client paths, we won't need this fstat
        self.ctx.p4.handler = FstatHandler(self.printed_revs, self.changes)
        fstat_cols = "-T" + ",".join(P4File.fstat_cols())
        self.ctx.p4.run("fstat", "-Of", fstat_cols, self._path_range())

        if self.graft_change:
            # Also run 'p4 fstat //<view>/...@change' for the graft
            # change to catch all files as of @change, not just
            # revs changed between begin and end of _path_range().
            self.ctx.p4.run("fstat", fstat_cols, self._graft_path())

        self.ctx.p4.handler = None

        self._collapse_to_graft_change()
        self._add_graft_to_changes()

        # don't need this any more
        self.printed_revs = None

        sorted_changes = [str(y) for y in sorted([int(x) for x in self.changes.keys()])]

        LOG.debug("\n".join([str(self.changes[ch]) for ch in sorted_changes]))
        return sorted_changes

    def _sync(self, sorted_changes):
        """fake sync of last change to make life easier at push time"""
        self.ctx.p4.handler = SyncHandler()
        lastchange = self.changes[sorted_changes[-1]]
        self.ctx.p4.run("sync", "-kf",
                self.ctx.client_view_path() + "@" + str(lastchange.change))
        self.ctx.p4.handler = None

    def _fast_import(self, sorted_changes, last_commit):
        """build fast-import script from changes, then run fast-import"""
        self.progress.progress_init_determinate(len(sorted_changes))
        for changenum in sorted_changes:
            change = self.changes[changenum]
            self.progress.progress_increment("Copying changelists...")
            self.ctx.heartbeat()

            # create commit and trees
            self.fastimport.add_commit(change, last_commit)

            last_commit = change.change

        # run git-fast-import and get list of marks
        marks = self.fastimport.run_fast_import()

        # done with these
        self.changes = None
        return marks

    def _mirror(self, marks):
        """build up list of p4 objects to mirror git repo in perforce
        then submit them
        """
        self.ctx.mirror.add_commits(marks)
        self.ctx.mirror.add_objects_to_p4(self.ctx)
        LOG.getChild("time").debug("\n\nGit Mirror:\n" + str(self.ctx.mirror))
        self.ctx.mirror = GitMirror(self.ctx.config.view_name)

        last_commit = marks[len(marks) - 1]
        LOG.debug("Last commit created: " + last_commit)

    # pylint: disable=R0201
    # R0201 Method could be a function
    def _pack(self):
        """run 'git gc' to pack up the blobs

        aside from any possible performance benefit, this prevents warnings
        from git about "unreachable loose objects"
        """
        p4gf_util.popen_no_throw(["git", "gc"])

    def _collapse_to_graft_change(self):
        """Move all of the files from pre-graft changelists into the graft
        changelist. Remove all pre-graft changelists.

        NOP if not grafting.

        'p4 print //client/...@100' does indeed print all the files that
        exist @100, but the tag dict that goes with each file includes the
        changelist in which that file was last added/edited, not 100. So
        this function gathers up all the file revs with change=1..99 and
        sticks them under change 100's file list.
        """
        if (not self.graft_change):
            return
        graft_num_int = int(self.graft_change.change)
        LOG.debug("_collapse_to_graft_change() graft_num_int={}".format(graft_num_int))

        # Delete all P4Changelist elements from self.changes where they
        # refer to a change that will be collapsed into the graft change,
        # including the graft change itself.
        del_keys = []
        for p4changelist in self.changes.values():
            if graft_num_int < int(p4changelist.change):
                LOG.debug("_collapse_to_graft_change() skipping {}".format(p4changelist.change))
                continue

            LOG.debug("_collapse_to_graft_change() deleting {}".format(p4changelist.change))
            del_keys.append(p4changelist.change)
        for key in del_keys:
            del self.changes[key]

        # Associate with the graft change all printed P4File results from
        # graft-change or older
        for (_key, p4file) in self.printed_revs.revs:
            if graft_num_int < int(p4file.change):
                LOG.debug("_collapse_to_graft_change() skipping post-graft {}".format(p4file))
                continue

            old = self.graft_change.file_from_depot_path(p4file.depot_path)
            # If print picked up multiple revs, keep the newest.
            if (not old) or (int(old.change) < int(p4file.change)):
                p4file.change = self.graft_change.change
                self.graft_change.files.append(p4file)
                LOG.debug("_collapse_to_graft_change() keeping {}".format(p4file))
            else:
                LOG.debug("_collapse_to_graft_change() skipping, had newer  {}".format(p4file))

    def _add_graft_to_changes(self):
        """Add the graft changelist to our list of changes:
        It will be copied over like any other change.

        NOP if not grafting.
        """
        if (not self.graft_change):
            return
        self.changes[self.graft_change.change] = self.graft_change

    def _graft_path(self):
        """If grafting, return '//<client>/...@N' where N is the graft
        changelist number.

        If not grafting, return None.
        """
        if (not self.graft_change):
            return
        return "{path}@{change}".format(
                        path = self.ctx.client_view_path(),
                        change = self.graft_change.change)

    def copy(self, start_at, stop_at):
        """copy a set of changelists from perforce into git"""

        with self.perf.timer[OVERALL]:
            with self.perf.timer[SETUP]:
                self._setup(start_at, stop_at)

                if not len(self.changes):
                    LOG.debug("No new changes found to copy")
                    return

                last_commit = self.rev_range.last_commit

            with self.perf.timer[PRINT]:
                self._copy_print()

            with self.perf.timer[FSTAT]:
                sorted_changes = self._fstat()

            with self.perf.timer[SYNC]:
                self._sync(sorted_changes)

            with self.perf.timer[FAST_IMPORT]:
                marks = self._fast_import(sorted_changes, last_commit)
                sorted_changes = None

            with self.perf.timer[MIRROR]:
                self._mirror(marks)

            with self.perf.timer[MERGE]:
                # merge temporary branch into master, then delete it
                self.fastimport.merge()

            with self.perf.timer[PACK]:
                self._pack()

        LOG.getChild("time").debug("\n" + str(self))