def _calc_repairs(ctx):
    """
    Scan Perforce for Git commit data and Perforce changelist descriptions,
    calculate which Perforce changelists need more data copied from Git
    backing store //.git-fusion/objects/...
    """

    # Load repo's entire set of Commit/Changelist metadata
    # into memory.
    LOG.info("Fetching list of Git commits/changelists from %s/objects/...",
             p4gf_const.objects_root())
    r = ctx.p4run(
        'files', '{root}/repos/{repo}/commits/...'.format(
            root=p4gf_const.objects_root(), repo=ctx.config.repo_name))
    # 'p4 print' each Git commit from its backup in
    # //.git-fusion/objects/...
    LOG.info("Fetched commit objects: {ct}".format(ct=len(r)))
    for rr in r:
        depot_path = rr.get('depotFile')
        if not depot_path:
            continue
        ot = ObjectType.commit_from_filepath(depot_path)
        SHA1_TO_OTL[ot.sha1].append(ot)
        LOG.debug('p4 print {}'.format(depot_path))
        blob_raw = p4gf_util.print_depot_path_raw(ctx.p4, depot_path)
        blob = p4gf_util.bytes_to_git_object(blob_raw)
        par_list = commit_to_parent_list(blob)
        SHA1_TO_PAR_SHA1_LIST[ot.sha1] = par_list
        LOG.debug("{sha1:7.7} parents={par}".format(
            sha1=ot.sha1, par=[p4gf_util.abbrev(p) for p in par_list]))

        # Loop through changelists, comparing against
        # backup and calculating if additional data
        # needs to be copied to its changelist description.
    return _calc_repairs_loop(ctx)
    def gc(self):   # pylint:disable=invalid-name
        """Do the thing."""
        if not self.cont:
            self.git_init()
            self.sql_init()
            self.sql_set_status(INIT)
            self.add_blobs_to_db()
            self.sql_set_status(BLOBS_ADDED)

            # add deleted trees to database
            depot_path = NTR("{}/trees/...".format(
                         p4gf_const.objects_root()))
            self.add_deleted_objects_to_db(depot_path, 'tree')
            self.sql_set_status(TREES_DELETED)

            # add commits to database
            depot_path = NTR("{}/repos/.../commits/...".format(
                         p4gf_const.objects_root()))
            self.add_deleted_objects_to_db(depot_path, 'commit')
            self.sql_set_status(BLOBS_DELETED)

            self.sync()
            self.sql_set_status(SYNC)
            # 'rename' the sync'ed files into .git/objects/xx/zzz...
            self.move_gf_objects_locations()
            pickled_value = pickle.dumps(self.table_names)
            # Save the table names and counts in the db
            # These will be read for the --cont option
            self.sql_set_admin(pickled_value, 'table_names')
            pickled_value = pickle.dumps(self.table_type_counts)
            self.sql_set_admin(pickled_value, 'table_type_counts')
            self.sql_set_status(OBJECTS_MOVED)
            self.db.commit()
        else:
            # will check the data base exists and get the status
            self.cont_init()

        if self.status == OBJECTS_MOVED:
            self.find_reachable()
            self.sql_set_status(REACHABLES_FOUND)
            self.db.commit()
        if self.status == REACHABLES_FOUND or (self.doreport and self.status == REPORTED):
            self.report()
        self.sql_set_status(REPORTED)
        self.db.commit()
        self.rmdir(keep=self.keep)
        laspsed_time = time.time() - self.starttime
        self.print_quiet(
                _("\nLapsed time: {}").format(str(datetime.timedelta(seconds=laspsed_time))))
Exemple #3
0
 def tree_p4_path(tree_sha1):
     '''
     Return depot path to a tree
     '''
     return (NTR('{objects_root}/trees/{slashed}').format(
         objects_root=p4gf_const.objects_root(),
         slashed=slashify_sha1(tree_sha1)))
Exemple #4
0
def generate_tags(ctx):
    """
    Regenerate the original tags into the (rebuilt) Git repository.
    This should only be called when the repository was just rebuilt
    from Perforce, otherwise it will do a bunch of work for nothing.
    """
    # Fetch everything under //.git-fusion/objects/repos/<repo>/tags/...
    tags_path = NTR('objects/repos/{repo}/tags').format(repo=ctx.config.view_name)
    with ctx.p4gf.at_exception_level(P4.P4.RAISE_NONE):
        client_path = "//{}/{}/...".format(ctx.config.p4client_gf, tags_path)
        ctx.p4gfrun(['sync', '-f', '-q', client_path])

    # Walk the tree looking for tags, reconstituting those we encounter.
    tags_root = os.path.join(ctx.gitlocalroot, tags_path)
    for walk_root, _, files in os.walk(tags_root):
        for name in files:
            fname = os.path.join(walk_root, name)
            _install_tag(ctx.view_repo, fname)

    # Update the tag change counter to avoid repeating our efforts.
    last_copied_change = _read_last_copied_tag(ctx)
    tags_path = '{root}/repos/{repo}/tags/...@{num},#head'.format(
        root=p4gf_const.objects_root(),
        repo=ctx.config.view_name,
        num=1 + int(last_copied_change))
    r = ctx.p4gfrun(['changes', '-m1', '-s', 'submitted', tags_path])
    changes = p4gf_util.first_dict(r)
    if changes:
        _write_last_copied_tag(ctx, changes['change'])
Exemple #5
0
def generate_tags(ctx):
    """
    Regenerate the original tags into the (rebuilt) Git repository.
    This should only be called when the repository was just rebuilt
    from Perforce, otherwise it will do a bunch of work for nothing.
    """
    # Fetch everything under //.git-fusion/objects/repos/<repo>/tags/...
    tags_path = NTR('objects/repos/{repo}/tags').format(
        repo=ctx.config.view_name)
    with ctx.p4gf.at_exception_level(P4.P4.RAISE_NONE):
        client_path = "//{}/{}/...".format(ctx.config.p4client_gf, tags_path)
        ctx.p4gfrun(['sync', '-f', '-q', client_path])

    # Walk the tree looking for tags, reconstituting those we encounter.
    tags_root = os.path.join(ctx.gitlocalroot, tags_path)
    for walk_root, _, files in os.walk(tags_root):
        for name in files:
            fname = os.path.join(walk_root, name)
            _install_tag(ctx.view_repo, fname)

    # Update the tag change counter to avoid repeating our efforts.
    last_copied_change = _read_last_copied_tag(ctx)
    tags_path = '{root}/repos/{repo}/tags/...@{num},#head'.format(
        root=p4gf_const.objects_root(),
        repo=ctx.config.view_name,
        num=1 + int(last_copied_change))
    r = ctx.p4gfrun(['changes', '-m1', '-s', 'submitted', tags_path])
    changes = p4gf_util.first_dict(r)
    if changes:
        _write_last_copied_tag(ctx, changes['change'])
 def __init__(self, gc, object_type):
     OutputHandler.__init__(self)
     self.gc = gc
     self.object_type = object_type
     self.message = _('Finding deleted %ss ' % (object_type,))
     if object_type == 'tree':
         self.depot_path_prefix = NTR("{}/trees/".format(
                      p4gf_const.objects_root()))
    def add_blobs_to_db(self):
        """Use p4 files to populate the database with blob sha1.
        The FilesHandler does the inserts and avoids memory overruns."""

        blobs_root = NTR('{objects_root}/blobs/').format(
                objects_root=p4gf_const.objects_root())
        num_blobs = self.get_num_files(blobs_root + '...')
        self.eta = p4gf_eta.ETA(total_ct = num_blobs)
        with ProgressReporter.Determinate(num_blobs):
            handler = FilesHandler(self, blobs_root)
            with self.p4.using_handler(handler):
                self.p4.run('files', blobs_root + '...')
Exemple #8
0
def _commit_p4_path(commit_sha1, changelist, repo, branch_id):
    '''
    Return depot path to a commit
    '''
    if commit_sha1 == '*':
        assert branch_id == '*' and changelist == '*'
        return (NTR('{objects_root}/repos/{repo}/commits/...').format(
            objects_root=p4gf_const.objects_root(), repo=repo))
    if branch_id == '*':
        assert changelist == '*'
        return (NTR('{objects_root}/repos/{repo}/commits/{slashed}-*').format(
            objects_root=p4gf_const.objects_root(),
            repo=repo,
            slashed=slashify_sha1(commit_sha1)))
    return (NTR(
        '{objects_root}/repos/{repo}/commits/{slashed}-{branch_id},{change}').
            format(objects_root=p4gf_const.objects_root(),
                   repo=repo,
                   slashed=slashify_sha1(commit_sha1),
                   branch_id=branch_id,
                   change=changelist))
Exemple #9
0
def tags_exist_in_cache(ctx):
    """Return boolean whether tags exist in object cache.
    """
    tags_path = '{root}/repos/{repo}/tags/...'.format(
        root=p4gf_const.objects_root(), repo=ctx.config.repo_name)
    r = ctx.p4run('files', '-e', '-m1', tags_path)
    for rr in r:
        if not isinstance(rr, dict):
            continue
        if 'depotFile' in rr:
            return True
    return False
Exemple #10
0
def commit_depot_path(commit_sha1, change_num, repo, branch_id):
    """Return depot path to a commit."""
    if commit_sha1 == '*':
        assert branch_id == '*' and change_num == '*'
        return (NTR('{objects_root}/repos/{repo}/commits/...')
                .format(objects_root=p4gf_const.objects_root(),
                        repo=repo))
    if branch_id == '*':
        assert change_num == '*'
        return (NTR('{objects_root}/repos/{repo}/commits/{slashed}-*')
                .format(objects_root=p4gf_const.objects_root(),
                        repo=repo,
                        slashed=p4gf_path.slashify_sha1(commit_sha1)))

    #    '{objects_root}/repos/{repo}/commits/{slashed}-{branch_id},{change_num}'
    return (DEPOT_PATH_PATTERN
            .format(objects_root=p4gf_const.objects_root(),
                    repo=repo,
                    slashed=p4gf_path.slashify_sha1(commit_sha1),
                    branch_id=branch_id,
                    change_num=change_num))
Exemple #11
0
def any_tags_since_last_copy(ctx):
    """
    Return True if there is at least one new change to the tags since the
    last time we copied between Git and Perforce.
    """
    last_copied_change = _read_last_copied_tag(ctx)
    tags_path = '{root}/repos/{repo}/tags/...@{num},#head'.format(
        root=p4gf_const.objects_root(),
        repo=ctx.config.view_name,
        num=1 + int(last_copied_change))
    r = ctx.p4gfrun(['changes', '-m1', tags_path])
    new = True if r else False
    LOG.debug('any_tags_since_last_copy() found new tags: {}'.format(new))
    return new
Exemple #12
0
def any_tags_since_last_copy(ctx):
    """
    Return True if there is at least one new change to the tags since the
    last time we copied between Git and Perforce.
    """
    last_copied_change = _read_last_copied_tag(ctx)
    tags_path = '{root}/repos/{repo}/tags/...@{num},#head'.format(
        root=p4gf_const.objects_root(),
        repo=ctx.config.view_name,
        num=1 + int(last_copied_change))
    r = ctx.p4gfrun(['changes', '-m1', tags_path])
    new = True if r else False
    LOG.debug('any_tags_since_last_copy() found new tags: {}'.format(new))
    return new
    def _accumulate_branch(self, branch):
        """Fetch lists of offending gitmirror files and p4keys,
        append to our accumulator data members
        """
        LOG.info("Check branch: {}"
                 .format(p4gf_util.abbrev(branch.branch_id)))

                        # Git commit objects in //.git-fusion/objects/...
        del_change_num_list = self._change_num_list_after(branch)
        self._log_results("Perforce changelist(s)", del_change_num_list)
        if not del_change_num_list:
            return
        del_depot_path_list = self._gitmirror_depot_path_list(
                                  branch
                                , del_change_num_list)
        self._log_results(_("Perforce copies of commit object(s)"
                            " in {root}/...".format(root=p4gf_const.objects_root())),
                          del_depot_path_list)
        self.del_depot_path_list.extend(del_depot_path_list)

                        # p4key index for each above commits.
        del_p4key_list = self._gitmirror_p4key_list(
                                  branch
                                , del_change_num_list)
        self._log_results("Perforce p4key(s) of commit object(s)", del_p4key_list)
        self.del_p4key_list.extend(del_p4key_list)

                        # Where to move each Git branch ref
                        # after rollback?
        if self.can_git:
            self.branch_to_surviving_ot[branch] \
                = self._git_branch_to_ot(branch)
            gbn_list = [b.git_branch_name
                        for b, ot in self.branch_to_surviving_ot.items()
                        if ot]
            self._log_results("Git reference(s)", gbn_list)

        if self.is_obliterate:
                        # Which changelists to obliterate?
            obli_change_num_list = self._change_num_list_to_obliterate(branch)
            self._log_results("changelist(s) to delete", obli_change_num_list)
            self.obliterate_change_num_list.extend(obli_change_num_list)

                        # Which files from those changelists to obliterate?
            obli_depot_rev_list = self._depot_rev_list_to_obliterate(
                                          branch
                                        , obli_change_num_list)
            self._log_results( "depot file revisions(s) to obliterate"
                             , obli_depot_rev_list )
            self.obliterate_depot_rev_set.update(obli_depot_rev_list)
Exemple #14
0
def update_tags(ctx):
    """Based on the recent changes to the tags, update our repository
    (remove deleted tags, add new pushed tags).
    """
    last_copied_change = _read_last_copied_tag(ctx)
    tags_prefix = '{root}/repos/{repo}/tags/'.format(
        root=p4gf_const.objects_root(), repo=ctx.config.repo_name)
    tags_path = '{prefix}...'.format(prefix=tags_prefix)
    num = 1 + int(last_copied_change)
    r = ctx.p4gfrun('changes', '-s', 'submitted', '-e', num, tags_path)
    changes = sorted(r, key=lambda k: int(k['change']))
    for change in changes:
        LOG.debug2('update_tags() processing {}'.format(change['change']))
        d = ctx.p4gfrun('describe', change['change'])
        d = p4gf_util.first_dict(d)
        for d_file, rev, action in zip(d['depotFile'], d['rev'], d['action']):
            LOG.debug2('update_tags() processing {} - {}'.format(
                d_file, action))
            if not d_file.startswith(tags_prefix):
                LOG.info('non-tag file {} in tag-related Git Fusion change {}'.
                         format(d_file, change['change']))
                continue
            if action == 'add':
                r = ctx.p4gfrun('sync', '-f', "{}#{}".format(d_file, rev))
                r = p4gf_util.first_dict(r)
                _install_tag(ctx.repo, r['clientFile'])
            elif action == 'delete':
                r = ctx.p4gfrun('sync', '-f',
                                "{}#{}".format(d_file,
                                               int(rev) - 1))
                r = p4gf_util.first_dict(r)
                _uninstall_tag(ctx.repo, r['clientFile'])
            elif action == 'edit':
                # get the tags named in the file prior to this change
                tags_before = _read_tags(ctx, d_file, int(rev) - 1)
                # get the tags named in the file after this change
                tags_after = _read_tags(ctx, d_file)
                # remove old (lightweight) tags and add new ones
                sha1 = d_file[-42:].replace('/', '')
                for old_tag in tags_before - tags_after:
                    _remove_tag_ref(old_tag, sha1)
                for new_tag in tags_after - tags_before:
                    _create_tag_ref(ctx.repo, new_tag, sha1)
            else:
                LOG.error(
                    "update_tags() received an unexpected change action: " +
                    "@{}, '{}' on {}".format(change['change'], action, d_file))
    _write_last_copied_tag(ctx, changes[-1]['change'])
Exemple #15
0
def update_tags(ctx):
    """
    Based on the recent changes to the tags, update our repository
    (remove deleted tags, add new pushed tags).
    """
    if not ctx.view_repo:
        # In some cases the Git repository object is not yet created.
        ctx.view_repo = pygit2.Repository(ctx.view_dirs.GIT_DIR)

    last_copied_change = _read_last_copied_tag(ctx)
    tags_path = '{root}/repos/{repo}/tags/...@{num},#head'.format(
        root=p4gf_const.objects_root(),
        repo=ctx.config.view_name,
        num=1 + int(last_copied_change))
    r = ctx.p4gfrun(['changes', '-s', 'submitted', tags_path])
    changes = sorted(r, key=lambda k: int(k['change']))
    for change in changes:
        d = ctx.p4gfrun(['describe', change['change']])
        d = p4gf_util.first_dict(d)
        for d_file, action in zip(d['depotFile'], d['action']):
            if action == 'add':
                r = ctx.p4gfrun(['sync', '-f', d_file])
                r = p4gf_util.first_dict(r)
                _install_tag(ctx.view_repo, r['clientFile'])
            elif action == 'delete':
                change_num = int(change['change']) - 1
                r = ctx.p4gfrun(
                    ['sync', '-f', "{}@{}".format(d_file, change_num)])
                r = p4gf_util.first_dict(r)
                _uninstall_tag(ctx.view_repo, r['clientFile'])
            elif action == 'edit':
                # get the tags named in the file prior to this change
                tags_before = _read_tags(ctx, d_file,
                                         int(change['change']) - 1)
                # get the tags named in the file after this change
                tags_after = _read_tags(ctx, d_file)
                # remove old (lightweight) tags and add new ones
                sha1 = d_file[-42:].replace('/', '')
                for old_tag in tags_before - tags_after:
                    _remove_tag_ref(old_tag, sha1)
                for new_tag in tags_after - tags_before:
                    _create_tag_ref(ctx.view_repo, new_tag, sha1)
            else:
                LOG.error(
                    "update_tags() received an unexpected change action: " +
                    "@{}, '{}' on {}".format(change['change'], action, d_file))
    _write_last_copied_tag(ctx, changes[-1]['change'])
 def _gitmirror_depot_path_list(self, branch, del_change_num_list):
     """Return a list of depot_paths, one for each undeleted gitmirror
     commit object on the given branch, in the given changelist number list.
     """
     result = []
     for del_change_num in del_change_num_list:
         depot_path = GITMIRROR_DEPOT_PATH.format(
               objects_root  = p4gf_const.objects_root()
             , repo          = self.ctx.config.repo_name
             , slashed       = "..."
             , branch_id     = branch.branch_id
             , change_num    = del_change_num
             )
         r = self.ctx.p4run('files', '-e', depot_path)
         for rr in r:
             if isinstance(rr, dict) and 'depotFile' in rr:
                 result.append(rr['depotFile'])
     return result
Exemple #17
0
def update_tags(ctx):
    """
    Based on the recent changes to the tags, update our repository
    (remove deleted tags, add new pushed tags).
    """
    if not ctx.view_repo:
        # In some cases the Git repository object is not yet created.
        ctx.view_repo = pygit2.Repository(ctx.view_dirs.GIT_DIR)

    last_copied_change = _read_last_copied_tag(ctx)
    tags_path = '{root}/repos/{repo}/tags/...@{num},#head'.format(
        root=p4gf_const.objects_root(),
        repo=ctx.config.view_name,
        num=1 + int(last_copied_change))
    r = ctx.p4gfrun(['changes', '-s', 'submitted', tags_path])
    changes = sorted(r, key=lambda k: int(k['change']))
    for change in changes:
        d = ctx.p4gfrun(['describe', change['change']])
        d = p4gf_util.first_dict(d)
        for d_file, action in zip(d['depotFile'], d['action']):
            if action == 'add':
                r = ctx.p4gfrun(['sync', '-f', d_file])
                r = p4gf_util.first_dict(r)
                _install_tag(ctx.view_repo, r['clientFile'])
            elif action == 'delete':
                change_num = int(change['change']) - 1
                r = ctx.p4gfrun(['sync', '-f', "{}@{}".format(d_file, change_num)])
                r = p4gf_util.first_dict(r)
                _uninstall_tag(ctx.view_repo, r['clientFile'])
            elif action == 'edit':
                # get the tags named in the file prior to this change
                tags_before = _read_tags(ctx, d_file, int(change['change']) - 1)
                # get the tags named in the file after this change
                tags_after = _read_tags(ctx, d_file)
                # remove old (lightweight) tags and add new ones
                sha1 = d_file[-42:].replace('/', '')
                for old_tag in tags_before - tags_after:
                    _remove_tag_ref(old_tag, sha1)
                for new_tag in tags_after - tags_before:
                    _create_tag_ref(ctx.view_repo, new_tag, sha1)
            else:
                LOG.error("update_tags() received an unexpected change action: " +
                          "@{}, '{}' on {}".format(change['change'], action, d_file))
    _write_last_copied_tag(ctx, changes[-1]['change'])
Exemple #18
0
def _find_client_commit_objects(args, p4, view_name):
    """Finds the object cache commit files associated only with the given view.
    These are objects that can be deleted from the cache without affecting
    other Git Fusion views. This does not return the eligible tree objects.

    Arguments:
        args -- parsed command line arguments
        p4 -- P4API object, client for object cache, already connected
        view_name -- name of view for which files are to be pruned from cache

    Returns:
        List of cached commit objects to be deleted.
    """

    # Bring the workspace up to date and traverse that rather than
    # fetching large numbers of small files from Perforce.
    repo_commit_objects_path = "{0}/repos/{1}/...".format(
        p4gf_const.objects_root(), view_name)
    repos_path = "{0}/repos/...".format(p4gf_const.P4GF_DEPOT)
    with p4.at_exception_level(P4.P4.RAISE_NONE):
        # Raises an exception when there are no files to sync?
        p4.run('sync', '-q', repo_commit_objects_path)
        p4.run('sync', '-q', repos_path)


# TBD Optimization:
# Rather than delete batches of files based on workspace file discovery
# we could do the following -- ??could overwhelm the server or be slower??
#   r = p4.run('delete', repo_commit_objects_path)
#   count = sum([int('depotFile' in rr and rr['action'] == 'delete') for rr in r])
#   r = p4.run("submit", "-d",
#            "Deleting {0} commit objects for repo '{1}'".format(count, view_name))
#   return count

    root = os.path.join(get_p4gf_localroot(p4), 'objects')
    print_verbose(
        args,
        _("Selecting cached commit objects for '{}'...").format(view_name))
    paths = [os.path.join(root, 'repos', view_name, '...')]
    return paths
Exemple #19
0
def any_tags_since_last_copy(ctx):
    """Return the first tags change not yet copied between Git and Perforce.

    If a tags change exists and is > last copied tags change, return it.
    Else return None.
    """
    last_copied_change = _read_last_copied_tag(ctx)
    if not last_copied_change:
        return None
    tags_path = '{root}/repos/{repo}/tags/...'.format(
        root=p4gf_const.objects_root(), repo=ctx.config.repo_name)
    head_change = p4gf_util.head_change_as_string(ctx,
                                                  submitted=True,
                                                  view_path=tags_path)

    if head_change and int(head_change) > int(last_copied_change):
        pass  # return this head_change
    else:
        head_change = None
    LOG.debug(
        'any_tags_since_last_copy() found new tags: {}'.format(head_change))
    return head_change
def _find_client_commit_objects(args, p4, view_name):
    """Finds the object cache commit files associated only with the given view.
    These are objects that can be deleted from the cache without affecting
    other Git Fusion views. This does not return the eligible tree objects.

    Arguments:
        args -- parsed command line arguments
        p4 -- P4API object, client for object cache, already connected
        view_name -- name of view for which files are to be pruned from cache

    Returns:
        List of cached commit objects to be deleted.
    """

    # Bring the workspace up to date and traverse that rather than
    # fetching large numbers of small files from Perforce.
    repo_commit_objects_path = "{0}/repos/{1}/...".format(p4gf_const.objects_root()
                ,view_name)
    repos_path = "{0}/repos/...".format(p4gf_const.P4GF_DEPOT)
    with p4.at_exception_level(P4.P4.RAISE_NONE):
        # Raises an exception when there are no files to sync?
        p4.run('sync', '-q', repo_commit_objects_path )
        p4.run('sync', '-q' , repos_path)

# TBD Optimization:
# Rather than delete batches of files based on workspace file discovery
# we could do the following -- ??could overwhelm the server or be slower??
#   r = p4.run('delete', repo_commit_objects_path)
#   count = sum([int('depotFile' in rr and rr['action'] == 'delete') for rr in r])
#   r = p4.run("submit", "-d",
#            "Deleting {0} commit objects for repo '{1}'".format(count, view_name))
#   return count

    root = os.path.join(get_p4gf_localroot(p4), 'objects')
    print_verbose(args,
            _("Selecting cached commit objects for '{}'...").format(view_name))
    paths = [os.path.join(root, 'repos', view_name, '...')]
    return paths
    def move_objects(self, walk_root, top_level_regex, subdir_regex, trim_hyphenated_suffix=False):
        """Move the object cache objects to '.git/objects'."""
        # pylint: disable=too-many-branches,too-many-statements
        # Because git paths are distributed over directory names taken
        # from the first 2 chars of the sha1, and Git Fusion cache
        # paths use xx/xx (trees and commits)
        # there is a boatload of data munging going on here.
        doing_trees = TREES in walk_root
        doing_commits = 'repos' in walk_root
        if doing_trees:
            progress_msg = _("Moving cached trees to local git ... {et} {ed}")
            object_count = self.sync_counts[TREES]
        else:
            progress_msg = _("Moving cached commits to local git ... {et} {ed}")
            object_count = self.sync_counts[COMMITS]
        self.eta = p4gf_eta.ETA(total_ct = object_count)
        with ProgressReporter.Determinate(object_count):
            for walk_root, _dirs, files in os.walk(walk_root):
                # For top level dirs, create the same dir under '.git/objects'
                m = top_level_regex.match(walk_root)
                if m:
                    for d in _dirs:
                        obj_dir = os.path.join(self.git_dir_objects, d)
                        p4gf_ensure_dir.ensure_dir(obj_dir)

                # If we have files we need to move them to 'git/objects'
                if files:
                    if not self.quiet:
                        self.eta.increment()
                        ProgressReporter.increment(progress_msg.
                            format( et = self.eta.eta_str()
                                  , ed = self.eta.eta_delta_str()))
                    sub1 = sub2 = None
                    m = subdir_regex.match(walk_root)
                    if m:
                        sub1 = m.group('sub1')
                        sub2 = m.group('sub2')
                    else:
                        LOG.error("regex failed to match as expected on {}.\nStopping.".format
                                (walk_root))
                        print("regex failed to match as expected on {}.\nStopping.".
                                format(walk_root))
                        sys.exit(1)
                    if doing_trees:
                        depot_path_prefix = NTR("{}/trees/{}/{}/".format(
                             p4gf_const.objects_root(), sub1, sub2))
                    for name in files:
                        git_file = sub2 + name
                        if trim_hyphenated_suffix:
                            git_file = re.sub(r'-.*$','',git_file)
                        git_sha1 = sub1 + git_file
                        if doing_trees:
                            depot_path = depot_path_prefix + name
                            self.sql_insert_object(TREES, git_sha1, depot_path)

                        git_path = os.path.join(self.git_dir_objects, sub1)
                        git_path = os.path.join(git_path,git_file)
                        p4_path = os.path.join(walk_root, name)
                        # Finally , move the p4 path to the git path
                        try:
                            os.rename(p4_path,git_path)
                        except OSError as e:
                            LOG.error("exception {}".format(str(e)))
                            sys.exit(1)
                        if doing_commits:
                            self.add_tree_from_commit_to_table(git_sha1)
                            # now that the commit's tree sha1 is in the db,
                            # the commit object is no longer needed
                            try:
                                os.unlink(git_path)
                            except OSError as e:
                                LOG.error("exception {}".format(str(e)))
                                sys.exit(1)
def blob_p4_path(blob_sha1):
    """Return depot path to a blob."""
    return (NTR('{objects_root}/blobs/{slashed}').format(
        objects_root=p4gf_const.objects_root(),
        slashed=slashify_blob_sha1(blob_sha1)))