def _pack(self):
        """run 'git gc' to pack up the blobs

        aside from any possible performance benefit, this prevents warnings
        from git about "unreachable loose objects"
        """
        p4gf_util.popen_no_throw(["git", "gc"])
Example #2
0
    def _pack(self):
        """run 'git gc' to pack up the blobs

        aside from any possible performance benefit, this prevents warnings
        from git about "unreachable loose objects"
        """
        p4gf_util.popen_no_throw(["git", "gc"])
def delete_empty_repo_branch(git_dir):
    '''
    Delete branch empty_repo. If we are currently on that branch,
    detach head before switching.

    Only do this if our HEAD points to an actual sha1: we have to have
    at least one commit.
    '''
    p4gf_util.popen_no_throw(['git', '--git-dir=' + git_dir, 'checkout', '-b', 'master'])
    p = p4gf_util.popen_no_throw(['git', '--git-dir=' + git_dir, 'branch', '--list',
            p4gf_const.P4GF_BRANCH_EMPTY_REPO])
    if p['out']:
        p = p4gf_util.popen_no_throw(['git', '--git-dir=' + git_dir, 'branch', '-D',
                p4gf_const.P4GF_BRANCH_EMPTY_REPO])
def _git_empty():
    """
    Is our git repo completely empty, not a single commit?
    """
    ### Replace git log with git rev-list.
    p = p4gf_util.popen_no_throw(['git', 'log', '-1', '--oneline'])
    return not p['out']
def create_git_repo(git_dir):
    """Create the git repository in the given root directory."""

    # Test if the Git repository has already been created.
    if os.path.exists(os.path.join(git_dir, 'HEAD')):
        return

    # Prepare the Git repository directory, cleaning up if necessary.
    if not os.path.exists(git_dir):
        parent = os.path.dirname(git_dir)
        if os.path.exists(parent):
            # weird case where git view dir exists but repo was deleted
            LOG.warn("mirror Git repository {} in bad state, repairing...".format(git_dir))
            shutil.rmtree(parent)
        LOG.debug("creating directory %s for Git repo", git_dir)
        os.makedirs(git_dir)

    # Initialize the Git repository for that directory.
    LOG.debug("creating Git repository in %s", git_dir)
    cmd = ['git', '--git-dir=' + git_dir, 'init']
    result = p4gf_util.popen_no_throw(cmd)
    if result['Popen'].returncode:
        code = result['Popen'].returncode
        LOG.error("error creating Git repo, git init returned %d", code)
        sys.stderr.write("error: git init failed with {} for {}\n".format(code, git_dir))

    create_empty_repo_branch(git_dir)
Example #6
0
def _git_empty():
    """
    Is our git repo completely empty, not a single commit?
    """
    ### Replace git log with git rev-list.
    p = p4gf_util.popen_no_throw(['git', 'log', '-1', '--oneline'])
    return not p['out']
Example #7
0
def create_git_repo(git_dir):
    """Create the git repository in the given root directory."""

    # Test if the Git repository has already been created.
    if os.path.exists(os.path.join(git_dir, 'HEAD')):
        return

    # Prepare the Git repository directory, cleaning up if necessary.
    if not os.path.exists(git_dir):
        parent = os.path.dirname(git_dir)
        if os.path.exists(parent):
            # weird case where git view dir exists but repo was deleted
            LOG.warn(
                "mirror Git repository {} in bad state, repairing...".format(
                    git_dir))
            shutil.rmtree(parent)
        LOG.debug("creating directory %s for Git repo", git_dir)
        os.makedirs(git_dir)

    # Initialize the Git repository for that directory.
    LOG.debug("creating Git repository in %s", git_dir)
    cmd = ['git', '--git-dir=' + git_dir, 'init']
    result = p4gf_util.popen_no_throw(cmd)
    if result['Popen'].returncode:
        code = result['Popen'].returncode
        LOG.error("error creating Git repo, git init returned %d", code)
        sys.stderr.write("error: git init failed with {} for {}\n".format(
            code, git_dir))

    create_empty_repo_branch(git_dir)
def copy_p2g(ctx, start):
    """Fill git with content from Perforce."""

    view_name = ctx.config.view_name
    view_dirs = ctx.view_dirs
    git_dir = view_dirs.GIT_DIR
    if not os.path.exists(git_dir):
        LOG.warn("mirror Git repository {} missing, recreating...".format(git_dir))
        # it's not the end of the world if the git repo disappears, just recreate it
        create_git_repo(git_dir)

    # If Perforce client view is empty and git repo is empty, someone is
    # probably trying to push into an empty repo/perforce tree. Let them.
    if _p4_empty(ctx) and _git_empty():
        LOG.info("Nothing to copy from empty view {}".format(view_name))
        return

    # We're not empty anymore, we no longer need this to avoid
    # git push rejection of push to empty repo refs/heads/master.
    delete_empty_repo_branch(view_dirs.GIT_DIR)

    start_at = p4gf_util.git_ref_master()
    if start and start_at:
        raise RuntimeError((  "Cannot use --start={start} when repo already has commits."
                            + " master={start_at}")
                           .format(start=start, start_at=start_at))
    if start:
        start_at = "@{}".format(start)
    elif start_at is None:
        start_at = "@1"

    p4gf_copy_to_git.copy_p4_changes_to_git(ctx, start_at, "#head")

    # Want to exit this function with HEAD and master both pointing to
    # the end of history. If we just copied anything from Perforce to
    # Git, point to the end of Perforce history.
    #
    # Want to leave this function with our temp branch gone. Otherwise
    # pullers will see "origin/git_fusion_temp_branch" and wonder "if
    # it's temp, why does it seem to live forever?"

    # Common: We have a temp branch that has added zero or more
    # commits ahead of master. Move HEAD and master to the temp branch's
    # commit. Move HEAD, detached (~0), first, just in case someone left it on master.
    temp_branch = p4gf_const.P4GF_BRANCH_TEMP + '~0'
    p1 = p4gf_util.popen_no_throw(['git', 'checkout', temp_branch])
    detached_head = (p1['Popen'].returncode == 0)
    if detached_head:
        p4gf_util.popen_no_throw(['git', 'branch', '-f', 'master', temp_branch])

    # Rare: If there are zero p4 changes in this view (yet), our temp
    # branch either does not exist or points nowhere and we were unable
    # to detach head from that temp branch. In that case switch to
    # (empty) branch master, creating it. We really want a master
    # branch, even if empty, so that we can delete the temp branch.
    if not detached_head:
        p4gf_util.popen_no_throw(['git', 'checkout', '-b', 'master'])

    p4gf_util.popen_no_throw(['git', 'branch', '-d', p4gf_const.P4GF_BRANCH_TEMP])
Example #9
0
def delete_empty_repo_branch(git_dir):
    '''
    Delete branch empty_repo. If we are currently on that branch,
    detach head before switching.

    Only do this if our HEAD points to an actual sha1: we have to have
    at least one commit.
    '''
    p4gf_util.popen_no_throw(
        ['git', '--git-dir=' + git_dir, 'checkout', '-b', 'master'])
    p = p4gf_util.popen_no_throw([
        'git', '--git-dir=' + git_dir, 'branch', '--list',
        p4gf_const.P4GF_BRANCH_EMPTY_REPO
    ])
    if p['out']:
        p = p4gf_util.popen_no_throw([
            'git', '--git-dir=' + git_dir, 'branch', '-D',
            p4gf_const.P4GF_BRANCH_EMPTY_REPO
        ])
Example #10
0
def expand_sha1(partial_sha1):
    """given partial SHA1 of a git object, return complete SHA1

    if there is no match, returns None
    """

    patt = re.compile(r'(?P<sha1>[0-9a-fA-F]{40,40}) (?P<type>[a-z]+) .*')
    cmd = ['git', 'cat-file', '--batch-check']
    result = p4gf_util.popen_no_throw(cmd, partial_sha1.encode())
    try:
        return patt.match(result['out']).group("sha1")
    except AttributeError:
        return None
def expand_sha1(partial_sha1):
    """given partial SHA1 of a git object, return complete SHA1

    if there is no match, returns None
    """

    patt = re.compile(r'(?P<sha1>[0-9a-fA-F]{40,40}) (?P<type>[a-z]+) .*')
    cmd = ['git', 'cat-file', '--batch-check']
    result = p4gf_util.popen_no_throw(cmd, partial_sha1.encode())
    try:
        return patt.match(result['out']).group("sha1")
    except AttributeError:
        return None
def ensure_deny_rewind(work_tree):
    """Initialize Git config with receive.denyNonFastForwards set to true.
    This prevents the Git user from rewinding our history and possibly
    leading to conflicting history if someone changes the history in
    Perforce at the same time (e.g. amends change descriptions).
    """
    cwd = os.getcwd()
    try:
        os.chdir(work_tree)
        cmd = ['git', 'config', '--local', '--replace-all', 'receive.denyNonFastForwards', 'true']
        result = p4gf_util.popen_no_throw(cmd)
        if result['Popen'].returncode:
            code = result['Popen'].returncode
            LOG.error("error configuring Git repo, git config returned %d", code)
            sys.stderr.write("error: git config failed with {} for {}\n".format(code, work_tree))
    finally:
        os.chdir(cwd)
Example #13
0
    def copy_commit(self, commit):
        """copy a single commit"""

        self._reset_for_new_commit()

        #OG.debug("dump commit {}".format(commit))
        LOG.debug("for  commit {}".format(commit['mark']))
        LOG.debug("with description: {}".format(commit['data']))
        LOG.debug("files affected: {}".format(commit['files']))

        # Reject merge commits. Not supported in 2012.1.
        if 'merge' in commit:
            self.revert_and_raise(("Merge commit {} not permitted." +
                                   " Rebase to create a linear" +
                                   " history.").format(commit['sha1']))

        # strip any enclosing angle brackets from the email address
        email = commit['author']['email'].strip('<>')
        user = self.usermap.lookup_by_email(email)
        LOG.debug("for email {} found user {}".format(email, user))
        if (user is None) or (not self.usermap.p4user_exists(user[0])):
            # User is not a known and existing Perforce user, and the
            # unknown_git account is not set up, so reject the commit.
            self.revert_and_raise(
                "User '{}' not permitted to commit".format(email))
        author_p4user = user[0]

        for blob in commit['files']:
            err = check_valid_filename(blob['path'])
            if err:
                self.revert_and_raise(err)

        with self.perf.timer[GIT_CHECKOUT]:
            d = p4gf_util.popen_no_throw(['git', 'checkout', commit['sha1']])
            if d['Popen'].returncode:
                # Sometimes git cannot distinquish the revision from a path...
                p4gf_util.popen(
                    ['git', 'reset', '--hard', commit['sha1'], '--'])

        with self.perf.timer[CHECK_PROTECTS]:
            self.check_protects(author_p4user, commit['files'])

        try:
            self.copy_blobs(commit['files'])
        except P4.P4Exception as e:
            self.revert_and_raise(str(e))

        with self.perf.timer[COPY_BLOBS_2]:
            pusher_p4user = self.ctx.authenticated_p4user
            LOG.debug("Pusher is: {}, author is: {}".format(
                pusher_p4user, author_p4user))
            desc = change_description(commit, pusher_p4user, author_p4user)

            try:
                opened = self.ctx.p4.run('opened')
                if opened:
                    changenum = p4_submit(self.ctx.p4, desc, author_p4user,
                                          commit['author']['date'])
                    LOG.info("Submitted change @{} for commit {}".format(
                        changenum, commit['sha1']))
                else:
                    LOG.info("Ignored empty commit {}".format(commit['sha1']))
                    return None
            except P4.P4Exception as e:
                self.revert_and_raise(str(e))
            return ":" + str(changenum) + " " + commit['sha1']
Example #14
0
def copy_p2g(ctx, start):
    """Fill git with content from Perforce."""

    view_name = ctx.config.view_name
    view_dirs = ctx.view_dirs
    git_dir = view_dirs.GIT_DIR
    if not os.path.exists(git_dir):
        LOG.warn(
            "mirror Git repository {} missing, recreating...".format(git_dir))
        # it's not the end of the world if the git repo disappears, just recreate it
        create_git_repo(git_dir)

    # If Perforce client view is empty and git repo is empty, someone is
    # probably trying to push into an empty repo/perforce tree. Let them.
    if _p4_empty(ctx) and _git_empty():
        LOG.info("Nothing to copy from empty view {}".format(view_name))
        return

    # We're not empty anymore, we no longer need this to avoid
    # git push rejection of push to empty repo refs/heads/master.
    delete_empty_repo_branch(view_dirs.GIT_DIR)

    start_at = p4gf_util.git_ref_master()
    if start and start_at:
        raise RuntimeError(
            ("Cannot use --start={start} when repo already has commits." +
             " master={start_at}").format(start=start, start_at=start_at))
    if start:
        start_at = "@{}".format(start)
    elif start_at is None:
        start_at = "@1"

    p4gf_copy_to_git.copy_p4_changes_to_git(ctx, start_at, "#head")

    # Want to exit this function with HEAD and master both pointing to
    # the end of history. If we just copied anything from Perforce to
    # Git, point to the end of Perforce history.
    #
    # Want to leave this function with our temp branch gone. Otherwise
    # pullers will see "origin/git_fusion_temp_branch" and wonder "if
    # it's temp, why does it seem to live forever?"

    # Common: We have a temp branch that has added zero or more
    # commits ahead of master. Move HEAD and master to the temp branch's
    # commit. Move HEAD, detached (~0), first, just in case someone left it on master.
    temp_branch = p4gf_const.P4GF_BRANCH_TEMP + '~0'
    p1 = p4gf_util.popen_no_throw(['git', 'checkout', temp_branch])
    detached_head = (p1['Popen'].returncode == 0)
    if detached_head:
        p4gf_util.popen_no_throw(
            ['git', 'branch', '-f', 'master', temp_branch])

    # Rare: If there are zero p4 changes in this view (yet), our temp
    # branch either does not exist or points nowhere and we were unable
    # to detach head from that temp branch. In that case switch to
    # (empty) branch master, creating it. We really want a master
    # branch, even if empty, so that we can delete the temp branch.
    if not detached_head:
        p4gf_util.popen_no_throw(['git', 'checkout', '-b', 'master'])

    p4gf_util.popen_no_throw(
        ['git', 'branch', '-d', p4gf_const.P4GF_BRANCH_TEMP])
Example #15
0
    def __init__(self, ctx,
                 testing_head_sha1=None,    # Only for testing, set to bypass
                                            # p4gf_object_type.sha1_to_object_type(),
                                            # set to -1 to leave self.good[] empty.
                 testing_head_change=None): # Only for testing
        self.ctx = ctx

        LOG.debug("checker init")
        # List of CommitChange tuples.
        #
        # As we copy from git to Perforce, record the most recent git
        # commit we just copied to Perforce, and the Perforce changelist
        # number that corresponds with that change. If we see any
        # changes other than this in our view, we know someone else has
        # submitted to our view and that we hit a conflict.
        #
        # element[0] is usally the git commit sha1 and Perforce
        # changelist number that correspond with HEAD at __init__ time.
        self.good                    = []

        # Updated by check(), point to element of good[] or None if no
        # conflict yet.
        self.first_conflict_index    = None

        # Updated by find_conflict_index() when no conflict is found.
        self.last_good_change_number = None

        # Dump out starting state.
        if LOG.isEnabledFor(logging.DEBUG):
            cmd = ['git', 'log', '--oneline', '--all', '--decorate', '-5']
            d = p4gf_util.popen_no_throw(cmd)
            LOG.debug('dumping current state:')
            LOG.debug('{}\n{}'.format(' '.join(cmd), d['out']))
            with ctx.p4.while_tagged(False):
                cmd = ['changes', '-m', '5', '-c', ctx.config.p4client]
                r = ctx.p4.run(cmd)
                LOG.debug('p4 {}\n{}'
                          .format( ' '.join(cmd)
                                 , '\n'.join(r)  ))

        if testing_head_sha1 != None or testing_head_change != None:
            # unit test hook to bypass p4gf_object_type.sha1_to_object_type()
            if testing_head_sha1 != -1:
                self.good.append(CommitChange(testing_head_sha1,
                                              testing_head_change))
        else:
            LOG.debug("checker getting head sha1")
            head_sha1 = p4gf_util.git_head_sha1()
            if head_sha1:
                object_type = p4gf_object_type.sha1_to_object_type(
                                          sha1      = p4gf_util.git_head_sha1()
                                        , view_name = ctx.config.view_name
                                        , p4        = ctx.p4)
                LOG.debug("checker got head sha1")
                if (object_type.type == 'commit'):
                    LOG.debug("checker got commit {}".format(object_type))
                    change_num = object_type.view_name_to_changelist(ctx.config.view_name)   
                    self.good.append(CommitChange(object_type.sha1, change_num))
                    self.last_good_change_number = change_num

        LOG.debug("end of __init__(): {}".format(self))
    def copy_commit(self, commit):
        """copy a single commit"""

        self._reset_for_new_commit()

        #OG.debug("dump commit {}".format(commit))
        LOG.debug("for  commit {}".format(commit['mark']))
        LOG.debug("with description: {}".format(commit['data']))
        LOG.debug("files affected: {}".format(commit['files']))

        # Reject merge commits. Not supported in 2012.1.
        if 'merge' in commit:
            self.revert_and_raise(("Merge commit {} not permitted."
                                   +" Rebase to create a linear"
                                   +" history.").format(commit['sha1']))

        # strip any enclosing angle brackets from the email address
        email = commit['author']['email'].strip('<>')
        user = self.usermap.lookup_by_email(email)
        LOG.debug("for email {} found user {}".format(email, user))
        if (user is None) or (not self.usermap.p4user_exists(user[0])):
            # User is not a known and existing Perforce user, and the
            # unknown_git account is not set up, so reject the commit.
            self.revert_and_raise("User '{}' not permitted to commit".format(email))
        author_p4user = user[0]

        for blob in commit['files']:
            err = check_valid_filename(blob['path'])
            if err:
                self.revert_and_raise(err)

        with self.perf.timer[GIT_CHECKOUT]:
            d = p4gf_util.popen_no_throw(['git', 'checkout', commit['sha1']])
            if d['Popen'].returncode:
                # Sometimes git cannot distinquish the revision from a path...
                p4gf_util.popen(['git', 'reset', '--hard', commit['sha1'], '--'])

        with self.perf.timer[CHECK_PROTECTS]:
            self.check_protects(author_p4user, commit['files'])

        try:
            self.copy_blobs(commit['files'])
        except P4.P4Exception as e:
            self.revert_and_raise(str(e))

        with self.perf.timer[COPY_BLOBS_2]:
            pusher_p4user = self.ctx.authenticated_p4user
            LOG.debug("Pusher is: {}, author is: {}".format(pusher_p4user, author_p4user))
            desc = change_description(commit, pusher_p4user, author_p4user)

            try:
                opened = self.ctx.p4.run('opened')
                if opened:
                    changenum = p4_submit(self.ctx.p4, desc, author_p4user,
                                          commit['author']['date'])
                    LOG.info("Submitted change @{} for commit {}".format(changenum, commit['sha1']))
                else:
                    LOG.info("Ignored empty commit {}".format(commit['sha1']))
                    return None
            except P4.P4Exception as e:
                self.revert_and_raise(str(e))
            return ":" + str(changenum) + " " + commit['sha1']