コード例 #1
0
    def _copy_print(self):
        """p4 print all revs and git-hash-object them into the git repo."""
        server_can_unexpand = self.ctx.p4.server_level > 32
        printhandler = PrintHandler(need_unexpand=not server_can_unexpand,
                                    tempdir=self.ctx.tempdir.name)
        self.ctx.p4.handler = printhandler
        args = ["-a"]
        if server_can_unexpand:
            args.append("-k")
        self.ctx.p4.run("print", args, self._path_range())
        printhandler.flush()
        printhandler.progress.progress_finish()

        # If also grafting, print all revs in existence at time of graft.
        if self.graft_change:
            args = []
            if server_can_unexpand:
                args.append("-k")
            path = self._graft_path()
            LOG.debug("Printing for grafted history: {}".format(path))
            self.ctx.p4.run("print", args, path)
            printhandler.flush()

            # If grafting, we just printed revs that refer to changelists
            # that have no P4Changelist counterpart in self.changes. Make
            # some skeletal versions now so that FstatHandler will have
            # someplace to hang its outputStat() P4File instances.
            for (_key, p4file) in printhandler.revs.revs:
                if not p4file.change in self.changes:
                    cl = P4Changelist()
                    cl.change = p4file.change
                    self.changes[p4file.change] = cl

        self.ctx.p4.handler = None
        self.printed_revs = printhandler.revs
コード例 #2
0
def get_keys_changes(p4, low, high):
    """Retrieve the set of changes made to the user keys between the two changes.

    Keyword arguments:
    p4   -- P4 API
    low  -- earliest change for which to retrieve changes
    high -- latest change for which to retrieve changes
    """
    rev_range = '@{},{}'.format(low, high)
    changes = P4Changelist.create_changelist_list_as_dict(
        p4, KEYS_PATH + rev_range)
    changes = sorted(changes.keys())
    root = '//{}/users'.format(p4gf_const.P4GF_DEPOT)
    changes = [
        P4Changelist.create_using_describe(p4, c, root) for c in changes
    ]
    return changes
コード例 #3
0
    def _setup(self, start_at, stop_at):
        """Set RevRange rev_range, figure out which changelists to copy."""
        self.rev_range = RevRange.from_start_stop(self.ctx, start_at, stop_at)
        LOG.debug(
            "Revision range to copy to Git: {rr}".format(rr=self.rev_range))

        # get list of changes to import into git
        self.changes = P4Changelist.create_changelist_list_as_dict(
            self.ctx.p4, self._path_range())

        # If grafting, get that too.
        if self.rev_range.graft_change_num:
            # Ignore all depotFile elements, we just want the change/desc/time/user.
            self.graft_change = P4Changelist.create_using_describe(
                self.ctx.p4, self.rev_range.graft_change_num,
                "ignore_depot_files")
            self.graft_change.description += (
                '\n[grafted history before {start_at}]'.format(
                    start_at=start_at))
コード例 #4
0
    def _setup(self, start_at, stop_at):
        """Set RevRange rev_range, figure out which changelists to copy."""
        self.rev_range = RevRange.from_start_stop(self.ctx, start_at, stop_at)
        LOG.debug("Revision range to copy to Git: {rr}"
                  .format(rr=self.rev_range))

        # get list of changes to import into git
        self.changes = P4Changelist.create_changelist_list_as_dict(
                            self.ctx.p4,
                            self._path_range())

        # If grafting, get that too.
        if self.rev_range.graft_change_num:
            # Ignore all depotFile elements, we just want the change/desc/time/user.
            self.graft_change = P4Changelist.create_using_describe(
                                    self.ctx.p4,
                                    self.rev_range.graft_change_num,
                                    "ignore_depot_files")
            self.graft_change.description += ('\n[grafted history before {start_at}]'
                                              .format(start_at=start_at))
コード例 #5
0
    def _copy_one(self, cnob):
        '''
        Copy one ChangeNumOnBranch element from Perforce to Git.

        p4 print all of its file revisions directly into .git/objects as blobs
        add them to the git-fast-import script
        '''
        _debug2('_copy_one {}', cnob)
        branch = self.ctx.branch_dict().get(cnob.branch_id)
        with self.ctx.switched_to_branch(branch):

            # Keep track of the highest changelist number we've
            # copied. Can't rely on
            # self.change_num_on_branch_list[-1] starting with our
            # highest changelist number because we might discover
            # new branches during later calls to _copy_one().
            change_num = int(cnob.change_num)
            if self.highest_copied_change_num < change_num:
                self.highest_copied_change_num = change_num
            self.cnob_count += 1

            # p4 changes -l -m1 @nnn
            #
            # Gets changelist description (including possible DescInfo),
            # owner, time.
            with self.p2g.perf.timer[CHANGES1]:
                r = self.ctx.p4run([
                    'changes',
                    '-l'  # include full changelist description
                    ,
                    '-m1'  # just this one changelist
                    ,
                    '@{}'.format(cnob.change_num)
                ])
                p4changelist = P4Changelist.create_using_changes(r[0])

            # p4 filelog -c nnnn -m1 //change_path/...
            #
            # Gets integration sources for parent calculations.
            # Gets files deleted at this rev (which 'p4 print' won't on 11.1).
            # Gets file list for this changelist.
            #
            # Cannot use p4 filelog //{client}/...@=nnn
            # That does request does not return one fstat for each file
            # in changelist nnn.
            with self.p2g.perf.timer[FILELOG]:
                cmd = ['filelog', '-c', cnob.change_num, '-m1', cnob.path]
                filelog_results = self.ctx.p4run(cmd)

            ### Detect lightweight integration sources not yet known.
            ### Create new Branch views to map them into this repo,
            ### run 'p4 changes' on them to add their history to our
            ### change_num_on_branch_list work queue, sorted.
            dbil = self.p2g.to_depot_branch_list_mc(change_num,
                                                    filelog_results)
            new_dbi_set = set(dbil) - self.known_dbi_set
            if new_dbi_set:
                ### push_front cnob
                for dbi in new_dbi_set:
                    LOG.error('AHA detected new integ source: {}'.format(dbi))
                    ### process dbi into branch, branch into more cnobs
                    ### mergesort new cnobs into cnob deque
                self.known_dbi_set.update(new_dbi_set)
                ### return, we'll deal with this later
                ### +++ save changes and filelog work

            # p4 print every revision modified by this changelist.
            #
            # +++ Also print every revision AFTER this changelist. There's a
            # +++ high probability that we'll need those revisons later.
            # +++ Printing them all now _greatly_ reduces the total number of
            # +++ 'p4 print' requests, reduces the Perforce server's workload
            # +++ (and thus repsponse time) in generating incremental file
            # +++ revisions from any files stored using RCS deltas (aka most
            # +++ files).
            with self.p2g.perf.timer[CALC_PRINT]:
                depot_path_rev_list = []
                for rr in filelog_results:
                    if ((not isinstance(rr, dict)) or ('depotFile' not in rr)
                            or ('rev' not in rr)):
                        continue
                    p4file = P4File.create_from_filelog(rr)
                    p4changelist.files.append(p4file)

                    depot_path = rr['depotFile']
                    rev = rr['rev'][0]
                    if self._already_printed(depot_path, rev):
                        continue
                    depot_path_rev_list.append('{}#{},head'.format(
                        depot_path, rev))

            rev_total = len(p4changelist.files)
            _debug2(
                'Printing files.'
                '  change: {change_num}'
                '  total: {rev_total}'
                '  need_print: {rev_need_print}'
                '  already_printed: {rev_already_printed}',
                change_num=cnob.change_num,
                rev_need_print=len(depot_path_rev_list),
                rev_already_printed=rev_total - len(depot_path_rev_list),
                rev_total=rev_total)

            if depot_path_rev_list:
                with self.p2g.perf.timer[PRINT2]:
                    printhandler = self._print_handler()
                    server_can_unexpand = self.ctx.p4.server_level > 32
                    args = ["-a"]
                    if server_can_unexpand:
                        args.append("-k")
                    cmd = ['print'] + args + depot_path_rev_list
                    with p4gf_util.RawEncoding(self.ctx.p4)             \
                    ,    p4gf_util.Handler(self.ctx.p4, printhandler)   \
                    ,    self.ctx.p4.at_exception_level(P4.RAISE_ALL):
                        self.ctx.p4run(cmd)
                    printhandler.flush()

            # Find each file revision's blob sha1.
            for p4file in p4changelist.files:
                symlink_path = _depot_rev_to_symlink(
                    depot_path=p4file.depot_path,
                    rev=p4file.revision,
                    symlink_dir=self.symlink_dir)
                blob_path = os.readlink(symlink_path)
                p4file.sha1 = _blob_path_to_sha1(blob_path)

            # If we can copy the Git commit and its tree objects from
            # our gitmirror, do so.
            # Non-MemCapped code calls all FI functions with
            # timer[FAST_IMPORT] as outer container, so must we.
            with self.p2g.perf.timer[FAST_IMPORT]:
                if self.p2g._fast_import_from_gitmirror(p4changelist, branch):
                    LOG.debug2('@{} fast-imported from gitmirror.'.format(
                        cnob.change_num))
                    return

                # Build a git-fast-import commit object.

                ### _fast_import_from_p4() runs its own filelog to
                ### discover integ sources. That needs to be hoisted up
                ### to our own filelog and passed down to avoid
                ### duplicate work.

                LOG.debug2('@{} fast-importing from p4 changelist.'.format(
                    cnob.change_num))
                self.p2g._fast_import_from_p4_mc(
                    change=p4changelist,
                    branch=branch,
                    filelog_results=filelog_results,
                    mark_to_branch_id=self.mark_to_branch_id,
                    branch_id_to_temp_name=self.branch_id_to_temp_name)
コード例 #6
0
    def _copy_one(self, cnob):
        '''
        Copy one ChangeNumOnBranch element from Perforce to Git.

        p4 print all of its file revisions directly into .git/objects as blobs
        add them to the git-fast-import script
        '''
        _debug2('_copy_one {}', cnob)
        branch = self.ctx.branch_dict().get(cnob.branch_id)
        with self.ctx.switched_to_branch(branch):

                        # Keep track of the highest changelist number we've
                        # copied. Can't rely on
                        # self.change_num_on_branch_list[-1] starting with our
                        # highest changelist number because we might discover
                        # new branches during later calls to _copy_one().
            change_num = int(cnob.change_num)
            if self.highest_copied_change_num < change_num:
                self.highest_copied_change_num = change_num
            self.cnob_count += 1

            # p4 changes -l -m1 @nnn
            #
            # Gets changelist description (including possible DescInfo),
            # owner, time.
            with self.p2g.perf.timer[CHANGES1]:
                r = self.ctx.p4run([ 'changes'
                                   , '-l'   # include full changelist description
                                   , '-m1'  # just this one changelist
                                   , '@{}'.format(cnob.change_num)])
                p4changelist = P4Changelist.create_using_changes(r[0])

            # p4 filelog -c nnnn -m1 //change_path/...
            #
            # Gets integration sources for parent calculations.
            # Gets files deleted at this rev (which 'p4 print' won't on 11.1).
            # Gets file list for this changelist.
            #
            # Cannot use p4 filelog //{client}/...@=nnn
            # That does request does not return one fstat for each file
            # in changelist nnn.
            with self.p2g.perf.timer[FILELOG]:
                cmd = [ 'filelog'
                      , '-c', cnob.change_num
                      , '-m1'
                      , cnob.path]
                filelog_results = self.ctx.p4run(cmd)

            ### Detect lightweight integration sources not yet known.
            ### Create new Branch views to map them into this repo,
            ### run 'p4 changes' on them to add their history to our
            ### change_num_on_branch_list work queue, sorted.
            dbil = self.p2g.to_depot_branch_list_mc(change_num, filelog_results)
            new_dbi_set = set(dbil) - self.known_dbi_set
            if new_dbi_set:
                ### push_front cnob
                for dbi in new_dbi_set:
                    LOG.error('AHA detected new integ source: {}'.format(dbi))
                    ### process dbi into branch, branch into more cnobs
                    ### mergesort new cnobs into cnob deque
                self.known_dbi_set.update(new_dbi_set)
                ### return, we'll deal with this later
                ### +++ save changes and filelog work

            # p4 print every revision modified by this changelist.
            #
            # +++ Also print every revision AFTER this changelist. There's a
            # +++ high probability that we'll need those revisons later.
            # +++ Printing them all now _greatly_ reduces the total number of
            # +++ 'p4 print' requests, reduces the Perforce server's workload
            # +++ (and thus repsponse time) in generating incremental file
            # +++ revisions from any files stored using RCS deltas (aka most
            # +++ files).
            with self.p2g.perf.timer[CALC_PRINT]:
                depot_path_rev_list = []
                for rr in filelog_results:
                    if (   (not isinstance(rr, dict))
                        or ('depotFile' not in rr)
                        or ('rev'       not in rr)):
                        continue
                    p4file = P4File.create_from_filelog(rr)
                    p4changelist.files.append(p4file)

                    depot_path = rr['depotFile']
                    rev        = rr['rev'][0]
                    if self._already_printed(depot_path, rev):
                        continue
                    depot_path_rev_list.append('{}#{},head'
                                               .format(depot_path, rev))

            rev_total = len(p4changelist.files)
            _debug2('Printing files.'
                    '  change: {change_num}'
                    '  total: {rev_total}'
                    '  need_print: {rev_need_print}'
                    '  already_printed: {rev_already_printed}'
                    , change_num          = cnob.change_num
                    , rev_need_print      = len(depot_path_rev_list)
                    , rev_already_printed = rev_total - len(depot_path_rev_list)
                    , rev_total           = rev_total)

            if depot_path_rev_list:
                with self.p2g.perf.timer[PRINT2]:
                    printhandler = self._print_handler()
                    server_can_unexpand = self.ctx.p4.server_level > 32
                    args = ["-a"]
                    if server_can_unexpand:
                        args.append("-k")
                    cmd = ['print'] + args + depot_path_rev_list
                    with p4gf_util.RawEncoding(self.ctx.p4)             \
                    ,    p4gf_util.Handler(self.ctx.p4, printhandler)   \
                    ,    self.ctx.p4.at_exception_level(P4.RAISE_ALL):
                        self.ctx.p4run(cmd)
                    printhandler.flush()

            # Find each file revision's blob sha1.
            for p4file in p4changelist.files:
                symlink_path = _depot_rev_to_symlink(
                                              depot_path  = p4file.depot_path
                                            , rev         = p4file.revision
                                            , symlink_dir = self.symlink_dir )
                blob_path   = os.readlink(symlink_path)
                p4file.sha1 = _blob_path_to_sha1(blob_path)

            # If we can copy the Git commit and its tree objects from
            # our gitmirror, do so.
                        # Non-MemCapped code calls all FI functions with
                        # timer[FAST_IMPORT] as outer container, so must we.
            with self.p2g.perf.timer[FAST_IMPORT]:
                if self.p2g._fast_import_from_gitmirror(p4changelist, branch):
                    LOG.debug2('@{} fast-imported from gitmirror.'
                               .format(cnob.change_num))
                    return

                # Build a git-fast-import commit object.

                        ### _fast_import_from_p4() runs its own filelog to
                        ### discover integ sources. That needs to be hoisted up
                        ### to our own filelog and passed down to avoid
                        ### duplicate work.

                LOG.debug2('@{} fast-importing from p4 changelist.'
                           .format(cnob.change_num))
                self.p2g._fast_import_from_p4_mc(
                      change                 = p4changelist
                    , branch                 = branch
                    , filelog_results        = filelog_results
                    , mark_to_branch_id      = self.mark_to_branch_id
                    , branch_id_to_temp_name = self.branch_id_to_temp_name )