Exemplo n.º 1
0
class Repository(BaseRepository):
    def __init__(self, *args, **kwargs):
        super(Repository, self).__init__(*args, **kwargs)

        self._repo = Repo(self.path)

    def _get_commit(self, commit_id):
        try:
            return self._repo.commit(commit_id)
        except Exception, e:
            raise CommitDoesNotExist("%s is not a commit" % commit_id)
Exemplo n.º 2
0
Arquivo: git.py Projeto: npryce/deft
class GitStorageHistory(object):
    def __init__(self, repodir, storage_root_subdir=os.curdir):
        self.repo = Repo(repodir)
        self.storage_root_subdir = storage_root_subdir
    
    def __getitem__(self, commit_sha):
        commit = self.repo.commit(commit_sha)
        tree = self.repo.tree(commit.tree)
        return GitTreeStorage(self.repo, tree, self.storage_root_subdir)
    
    @property
    def latest_revision(self):
        return self.repo.head()
    
    def eod_revisions(self, max_revision=None):
        if max_revision is None:
            max_revision = self.latest_revision
        
        results = {}
        commit_shas = set([max_revision])
        
        while commit_shas:
            commit_sha = commit_shas.pop()
            commit = self.repo.commit(commit_sha)
            commit_date = date_of(commit)
            
            commit_info = (time_of(commit), commit_sha)
            
            if commit_date in results:
                results[commit_date] = max(results[commit_date], commit_info)
            else:
                results[commit_date] = commit_info
            
            commit_shas.update(commit.parents)
        
        return dict((date, sha) for (date, (time, sha)) in results.iteritems())
Exemplo n.º 3
0
def get_commit_details():
    """
    If there's a '.git' directory besides 'dreampielib', return the current
    commit (HEAD) id and commit time.
    Otherwise, return None, None.
    """
    git_dir = join(dirname(dirname(dirname(abspath(__file__)))), '.git')
    if not isdir(git_dir):
        return None, None
    try:
        from dulwich.repo import Repo
    except ImportError:
        return None, None
    
    repo = Repo(git_dir)
    commit_id = repo.refs['HEAD']
    commit = repo.commit(commit_id)
    return commit_id, commit.commit_time
Exemplo n.º 4
0
def git_prelod(abbr):
    if not hasattr(settings, "ENABLE_GIT") or not settings.ENABLE_GIT:
        return

    global git_active_repo
    global git_active_commit
    global git_active_tree
    global git_old_tree
    global HEAD

    gitdir = "%s/%s.git" % (settings.GIT_PATH, abbr)

    if not os.path.exists(gitdir):
        git_repo_init(gitdir)

    git_active_repo = Repo(gitdir)
    git_active_commit = Commit()
    HEAD = git_active_repo.head()
    commit = git_active_repo.commit(HEAD)
    tree = git_active_repo.tree(commit.tree)
    git_old_tree = tree.id
    git_active_tree = tree
Exemplo n.º 5
0
def git_prelod(abbr):
    if not hasattr(settings, "ENABLE_GIT") or not settings.ENABLE_GIT:
        return

    global git_active_repo
    global git_active_commit
    global git_active_tree
    global git_old_tree
    global HEAD

    gitdir = "%s/%s.git" % (settings.GIT_PATH, abbr)

    if not os.path.exists(gitdir):
        git_repo_init(gitdir)

    git_active_repo = Repo(gitdir)
    git_active_commit = Commit()
    HEAD = git_active_repo.head()
    commit = git_active_repo.commit(HEAD)
    tree = git_active_repo.tree(commit.tree)
    git_old_tree = tree.id
    git_active_tree = tree
Exemplo n.º 6
0
    def __init__(self, path, init = False):
        if init :
          os.makedirs(path)
          repo = Repo.init_bare(path)
        else:
          repo = Repo(path)


        self._repo    = repo
        self.path     = path
        self.branches = self._getBranches() 

        try:
            head = repo.head()
            commitLast = repo.commit(head)
            
            self.head     = {
                'sha'    : head,
                'tree'   : commitLast.tree
            }
        except KeyError:
            self.head = None
Exemplo n.º 7
0
def filetree(request, repo_name, sha, path=''):
    try:
        repo = Repo(os.path.join(settings.REPOS_DIR, repo_name))
    except NotGitRepository:
        raise Http404
    
    # this may be needed, depnding on how we do branches
    #branch = getBranch(request)
    
    # check to see if a specific commit has been passed
    # if not, then assume head

    sha_ref = 'refs/heads/' + sha 
    if sha_ref in repo.refs:
        sha = repo.refs[sha_ref]
       
    #raise Exception('%s - %s' % (sha, path))
    commit = repo.commit(sha)
    
    tree = repo.tree(commit.tree)
    
    # Lookup the path; may be a tree or a blob
    obj = tree.lookup_path(repo.get_object, path)[1]
    
    try: 
        # assume it's a tree
        new_tree = repo.tree(obj)
    except NotTreeError:
        # well maybe it was a file (blob)
        return file_(request, repo_name, repo, sha=obj, path=path)
        
    files = util.getFiles(new_tree, repo)
    
    return render_to_response("filetree.html",
                              dict(name=repo_name,
                                   files=files,
                                   branch='master',
                                   path=path))
Exemplo n.º 8
0
def filetree(request, repo_name, sha, path=''):
    try:
        repo = Repo(os.path.join(settings.REPOS_DIR, repo_name))
    except NotGitRepository:
        raise Http404

    # this may be needed, depnding on how we do branches
    #branch = getBranch(request)

    # check to see if a specific commit has been passed
    # if not, then assume head

    sha_ref = 'refs/heads/' + sha
    if sha_ref in repo.refs:
        sha = repo.refs[sha_ref]

    #raise Exception('%s - %s' % (sha, path))
    commit = repo.commit(sha)

    tree = repo.tree(commit.tree)

    # Lookup the path; may be a tree or a blob
    obj = tree.lookup_path(repo.get_object, path)[1]

    try:
        # assume it's a tree
        new_tree = repo.tree(obj)
    except NotTreeError:
        # well maybe it was a file (blob)
        return file_(request, repo_name, repo, sha=obj, path=path)

    files = util.getFiles(new_tree, repo)

    return render_to_response(
        "filetree.html",
        dict(name=repo_name, files=files, branch='master', path=path))
Exemplo n.º 9
0
class GitStorage():
    def _ignoreFile(self, dirName, fileName):
        """
        used for the copTree stuff
        ``dirName``
            the working directory
        ``fileName``
            list of files inside the directory (dirName)
        """
        result = []
        for i in fileName:
            path = dirName + i
            if path not in fileToIgnore:
                result.append(path)
        return result

    def _commit(self, tree):
        """
        commit a tree used only by the init
        ``tree``
            tree to commit
        """
        commit = Commit()
        commit.tree = tree.id
        commit.encoding = "UTF-8"
        commit.committer = commit.author = 'debexpo <%s>' % (pylons.config['debexpo.email'])
        commit.commit_time = commit.author_time = int(time())
        tz = parse_timezone('-0200')[0]
        commit.commit_timezone = commit.author_timezone = tz
        commit.message = " "
        self.repo.object_store.add_object(tree)
        self.repo.object_store.add_object(commit)
        self.repo.refs["HEAD"] = commit.id
        log.debug('commiting')
        return commit.id

    def __init__(self, path):
        #creating the repository
        if os.path.isdir(path):
            log.debug("directory exist, taking it as a git repository")
            self.repo = Repo(path)
        else:
            log.debug("directory doesn't exist, creating")
            os.makedirs(path)
            log.debug("initiate the repo")
            self.repo = Repo.init(path)
            log.debug("adding an empty tree to the repository")
            self._commit(Tree())

    #only this function will be used on upload
    def change(self, files):
        """
        used to change  afile in the git storage can be called for the first upload we don't care
        ``files``
            a list of file to change
        """
        if len(files) == 0:
            log.debug("trying to change nothing will do... nothing")
        else:
            log.debug("this will change %i files" % (len(files)))
            for f in files:
                self.repo.stage(str(f))
            log.debug("stages dones")
            self.repo.do_commit("this is so awesome that nobody will never see it",
                committer="same here <*****@*****.**>")

    def buildTreeDiff(self, dest, tree=None, originalTree=None):
        """
        creating files from the diff between 2 trees, it will be used in the code browser to get older version
        (walking on history)
        ``tree``
            the tree that you want to compare to
        ``dest``
            the destination folder to build sources
        ``originalTree``
            the original Tree, by default it's the last one

        by default it's retun the last changed files

        """
        if tree is None:
            head = self.repo.commit(self.repo.commit(self.repo.head()).parents[0])
            tree = self.repo.tree(head.tree)
        if originalTree is None:
            originalTree = self.repo.tree(self.repo.commit(self.repo.head()).tree)
        blobToBuild = []
        #getting blob that have changed
        for blob in self.repo.object_store.iter_tree_contents(tree.id):
            if blob not in originalTree:
                blobToBuild.append(blob)
                fileToIgnore.append(blob.path)
        repoLocation = os.path.join(str(self.repo).split("'")[1])
        #creating the folder with link to older files
        if os.path.exists(repoLocation + dest):
            log.warning("%s already exist, copy will not work")
        else:
            log.debug("copying files")
            shutil.copytree(repoLocation, repoLocation + dest, symlinks=True, ignore=self._ignoreFile)
        for b in blobToBuild:
            fileDirectory = os.path.split(b.path)
            fileDirectory.pop()
            if not os.path.exists(os.path.join(repoLocation + dest, os.path.join(fileDirectory))):
                os.makedirs(os.path.join(repoLocation + dest, os.path.join(fileDirectory)))
            file = open(os.path.join(repoLocation + dest, b.path), 'w')
            file.write(self.repo.get_object(b.sha).as_raw_string())
            file.close()
        tree = None
        originalTree = None

    #get*
    def getLastTree(self):
        """
        return the last tree
        """
        return self.repo.tree(self.repo._commit(self.repo.head()).tree)

    def getAllTrees(self):
        """
        return trees
        """
        result = []
        commit = self.repo._commit(self.repo.head())
        for c in commit._get_parents():
            result.append(c.tree)
        return result

    def getOlderFileContent(self, file):
        """
        return the first file's content that changed from the file
        ``file``
            the file to work on
        """
        with open(file) as f:
            originalBlob = Blob.from_string("".join(f.readlines()))
        trees = self.getAllTrees()
        for t in trees:
            #parsing tree in order to find the tree where the file change
            if originalBlob not in t:
                tree = t
                break
                #tree must be existent, other way file is not correct
        if tree is None:
            log.error(
                "there is no tree that contain this blob this souldn't happen, other way this file does not appear to come from this package")
        else:
            if self.repo._commit(self.repo.head()).tree == tree:
                olderTree = self.repo.commit(self.repo.head())._get_parents()[0].tree
            else:
                for c in self.repo._commit(self.repo.head())._get_parents():
                    if c.tree == tree:
                        try:
                            olderTree = c.get_parents()[0]
                        except IndexError:
                            log.debug("file is the last version")
                            olderTree = tree
            if olderTree != tree:
                #we must check here the blob that contains the older file
                for b in self.repo.object_store.iter_tree_contents(olderTree.id):
                    if originalBlob.path == b.path:
                        #older blob find! awesome, in the first loop we already test if they are the same
                        # that's why we can now return the content of the file
                        return self.repo.get_object(b.sha).as_raw_string()
        return ""

    def getOlderCommits(self):
        """
        return a list of all commits
        """
        return self.repo.commit(self.repo.head())._get_parents()
Exemplo n.º 10
0
class GitHandler(object):

    def __init__(self, dest_repo, ui):
        self.repo = dest_repo
        self.ui = ui
        self.mapfile = 'git-mapfile'
        self.configfile = 'git-config'

        if ui.config('git', 'intree'):
            self.gitdir = self.repo.wjoin('.git')
        else:
            self.gitdir = self.repo.join('git')

        self.importbranch = ui.config('git', 'importbranch')
        self.exportbranch = ui.config('git', 'exportbranch', 'refs/heads/master')
        self.bookbranch = ui.config('git', 'bookbranch', '')

        self.init_if_missing()
        self.load_git()
        self.load_map()
        self.load_config()

    # make the git data directory
    def init_if_missing(self):
        if not os.path.exists(self.gitdir):
            os.mkdir(self.gitdir)
            Repo.init_bare(self.gitdir)

    def load_git(self):
        self.git = Repo(self.gitdir)

    ## FILE LOAD AND SAVE METHODS

    def map_set(self, gitsha, hgsha):
        self._map_git[gitsha] = hgsha
        self._map_hg[hgsha] = gitsha

    def map_hg_get(self, gitsha):
	return self._map_git.get(gitsha)

    def map_git_get(self, hgsha):
	return self._map_hg.get(hgsha)

    def load_map(self):
        self._map_git = {}
        self._map_hg = {}
        if os.path.exists(self.repo.join(self.mapfile)):
            for line in self.repo.opener(self.mapfile):
                gitsha, hgsha = line.strip().split(' ', 1)
                self._map_git[gitsha] = hgsha
                self._map_hg[hgsha] = gitsha

    def save_map(self):
        file = self.repo.opener(self.mapfile, 'w+', atomictemp=True)
        for gitsha, hgsha in sorted(self._map_git.iteritems()):
            file.write("%s %s\n" % (gitsha, hgsha))
        file.rename()

    def load_config(self):
        self._config = {}
        if os.path.exists(self.repo.join(self.configfile)):
            for line in self.repo.opener(self.configfile):
                key, value = line.strip().split(' ', 1)
                self._config[key] = value

    def save_config(self):
        file = self.repo.opener(self.configfile, 'w+', atomictemp=True)
        for key, value in self._config.iteritems():
            file.write("%s %s\n" % (key, value))
        file.rename()


    ## END FILE LOAD AND SAVE METHODS

    def import_commits(self, remote_name):
        self.import_git_objects(remote_name)
        self.save_map()

    def fetch(self, remote_name):
        self.ui.status(_("fetching from : %s\n") % remote_name)
        self.export_git_objects()
        refs = self.fetch_pack(remote_name)
        if refs:
            self.import_git_objects(remote_name, refs)
            self.import_local_tags(refs)
        self.save_map()

    def export_commits(self):
        self.export_git_objects()
        self.export_hg_tags()
        self.update_references()
        self.save_map()

    def push(self, remote_name):
        self.ui.status(_("pushing to : %s\n") % remote_name)
        self.export_commits()
        self.update_remote_references(remote_name)
        self.upload_pack(remote_name)

    def remote_add(self, remote_name, git_url):
        self._config['remote.' + remote_name + '.url'] = git_url
        self.save_config()

    def remote_remove(self, remote_name):
        key = 'remote.' + remote_name + '.url'
        if key in self._config:
            del self._config[key]
        self.save_config()

    def remote_show(self, remote_name):
        key = 'remote.' + remote_name + '.url'
        if key in self._config:
            name = self._config[key]
            self.ui.status(_("URL for %s : %s\n") % (remote_name, name, ))
        else:
            self.ui.status(_("No remote named : %s\n") % remote_name)
        return

    def remote_list(self):
        for key, value in self._config.iteritems():
            if key[0:6] == 'remote':
                self.ui.status('%s\t%s\n' % (key, value, ))

    def remote_name_to_url(self, remote_name):
        return self._config['remote.' + remote_name + '.url']

    def update_references(self):
        try:
            # We only care about bookmarks of the form 'name',
            # not 'remote/name'.
            def is_local_ref(item): return item[0].count('/') == 0
            bms = bookmarks.parse(self.repo)
            bms = dict(filter(is_local_ref, bms.items()))

            # Create a local Git branch name for each
            # Mercurial bookmark.
            for key in bms:
                hg_sha  = hex(bms[key])
                git_sha = self.map_git_get(hg_sha)
                self.git.set_ref('refs/heads/' + key, git_sha)
        except AttributeError:
            # No bookmarks extension
            pass

        c = self.map_git_get(hex(self.repo.changelog.tip()))
        self.git.set_ref(self.exportbranch, c)

    def export_hg_tags(self):
        for tag, sha in self.repo.tags().iteritems():
            if tag[-3:] == '^{}':
                continue
            if tag == 'tip':
                continue 
            self.git.set_ref('refs/tags/' + tag, self.map_git_get(hex(sha)))

    # Make sure there's a refs/remotes/remote_name/name
    #           for every refs/heads/name
    def update_remote_references(self, remote_name):
        self.git.set_remote_refs(self.local_heads(), remote_name)

    def local_heads(self):
        def is_local_head(item): return item[0].startswith('refs/heads')
        refs = self.git.get_refs()
        return dict(filter(is_local_head, refs.items()))

    def export_git_objects(self):
        self.ui.status(_("importing Hg objects into Git\n"))
        total = len(self.repo.changelog)
        if total:
          magnitude = int(math.log(total, 10)) + 1
        else:
          magnitude = 1
        for i, rev in enumerate(self.repo.changelog):
            if i%100 == 0:
                self.ui.status(_("at: %*d/%d\n") % (magnitude, i, total))
            
            ctx = self.repo.changectx(rev)
            state = ctx.extra().get('hg-git', None)
            if state == 'octopus':
                self.ui.debug("revision %d is a part of octopus explosion\n" % rev)
                continue
            pgit_sha, already_written = self.export_hg_commit(rev)
            if not already_written:
                self.save_map()

    # convert this commit into git objects
    # go through the manifest, convert all blobs/trees we don't have
    # write the commit object (with metadata info)
    def export_hg_commit(self, rev):
        def is_octopus_part(ctx):
            return ctx.extra().get('hg-git', None) in set(['octopus', 'octopus-done'])

        # return if we've already processed this
        node = self.repo.changelog.lookup(rev)
        phgsha = hex(node)
        pgit_sha = self.map_git_get(phgsha)
        if pgit_sha:
            return pgit_sha, True

        self.ui.status(_("converting revision %s\n") % str(rev))

        # make sure parents are converted first
        ctx = self.repo.changectx(rev)
        extra = ctx.extra()

        parents = []
        if extra.get('hg-git', None) == 'octopus-done':
            # implode octopus parents
            part = ctx
            while is_octopus_part(part):
                (p1, p2) = part.parents()
                assert not is_octopus_part(p1)
                parents.append(p1)
                part = p2
            parents.append(p2)
        else:
            parents = ctx.parents()

        for parent in parents:
            p_rev = parent.rev()
            hgsha = hex(parent.node())
            git_sha = self.map_git_get(hgsha)
            if not p_rev == -1:
                if not git_sha:
                    self.export_hg_commit(p_rev)

        tree_sha, renames = self.write_git_tree(ctx)
        
        commit = {}
        commit['tree'] = tree_sha
        (time, timezone) = ctx.date()

        # hg authors might not have emails
        author = ctx.user()
        if not '>' in author: 
            author = author + ' <none@none>'
        commit['author'] = author + ' ' + str(int(time)) + ' ' + format_timezone(-timezone)
        message = ctx.description()
        commit['message'] = ctx.description() + "\n"

        if 'committer' in extra:
            # fixup timezone
            (name_timestamp, timezone) = extra['committer'].rsplit(' ', 1)
            try:
                timezone = format_timezone(-int(timezone))
                commit['committer'] = '%s %s' % (name_timestamp, timezone)
            except ValueError:
                self.ui.warn(_("Ignoring committer in extra, invalid timezone in r%s: '%s'.\n") % (rev, timezone))
        if 'encoding' in extra:
            commit['encoding'] = extra['encoding']

        # HG EXTRA INFORMATION
        add_extras = False
        extra_message = ''
        if not ctx.branch() == 'default':
            add_extras = True
            extra_message += "branch : " + ctx.branch() + "\n"

        if renames:
            add_extras = True
            for oldfile, newfile in renames:
                extra_message += "rename : " + oldfile + " => " + newfile + "\n"

        for key, value in extra.iteritems():
            if key in ['committer', 'encoding', 'branch', 'hg-git', 'git']:
                continue
            else:
                add_extras = True        
                extra_message += "extra : " + key + " : " +  urllib.quote(value) + "\n"

        if add_extras:
            commit['message'] += "\n--HG--\n" + extra_message

        commit['parents'] = []
        for parent in parents:
            hgsha = hex(parent.node())
            git_sha = self.map_git_get(hgsha)
            if git_sha:
                commit['parents'].append(git_sha)

        commit_sha = self.git.write_commit_hash(commit) # writing new blobs to git
        self.map_set(commit_sha, phgsha)
        return commit_sha, False

    def write_git_tree(self, ctx):
        trees = {}
        man = ctx.manifest()
        renames = []
        for filenm in man.keys():
            # write blob if not in our git database
            fctx = ctx.filectx(filenm)
            rename = fctx.renamed()
            if rename:
                filerename, sha = rename
                renames.append((filerename, filenm))
            is_exec = 'x' in fctx.flags()
            is_link = 'l' in fctx.flags()
            file_id = hex(fctx.filenode())
            blob_sha = self.map_git_get(file_id)
            if not blob_sha:
                blob_sha = self.git.write_blob(fctx.data()) # writing new blobs to git
                self.map_set(blob_sha, file_id)

            parts = filenm.split('/')
            if len(parts) > 1:
                # get filename and path for leading subdir
                filepath = parts[-1:][0]
                dirpath = "/".join([v for v in parts[0:-1]]) + '/'

                # get subdir name and path for parent dir
                parpath = '/'
                nparpath = '/'
                for part in parts[0:-1]:
                    if nparpath == '/':
                        nparpath = part + '/'
                    else:
                        nparpath += part + '/'

                    treeentry = ['tree', part + '/', nparpath]

                    if parpath not in trees:
                        trees[parpath] = []
                    if treeentry not in trees[parpath]:
                        trees[parpath].append( treeentry )

                    parpath = nparpath

                # set file entry
                fileentry = ['blob', filepath, blob_sha, is_exec, is_link]
                if dirpath not in trees:
                    trees[dirpath] = []
                trees[dirpath].append(fileentry)

            else:
                fileentry = ['blob', parts[0], blob_sha, is_exec, is_link]
                if '/' not in trees:
                    trees['/'] = []
                trees['/'].append(fileentry)

        dirs = trees.keys()
        if dirs:
            # sort by tree depth, so we write the deepest trees first
            dirs.sort(lambda a, b: len(b.split('/'))-len(a.split('/')))
            dirs.remove('/')
            dirs.append('/')
        else:
            # manifest is empty => make empty root tree
            trees['/'] = []
            dirs = ['/']

        # write all the trees
        tree_sha = None
        tree_shas = {}
        for dirnm in dirs:
            tree_data = []
            for entry in trees[dirnm]:
                # replace tree path with tree SHA
                if entry[0] == 'tree':
                    sha = tree_shas[entry[2]]
                    entry[2] = sha
                tree_data.append(entry)
            tree_sha = self.git.write_tree_array(tree_data) # writing new trees to git
            tree_shas[dirnm] = tree_sha
        
        return (tree_sha, renames) # should be the last root tree sha

    def remote_head(self, remote_name):
        for head, sha in self.git.remote_refs(remote_name).iteritems():
            if head == 'HEAD':
                return self.map_hg_get(sha)
        return None

    def upload_pack(self, remote_name):
        git_url = self.remote_name_to_url(remote_name)
        client, path = self.get_transport_and_path(git_url)
        changed = self.get_changed_refs
        genpack = self.generate_pack_contents
        try:
            self.ui.status(_("creating and sending data\n"))
            changed_refs = client.send_pack(path, changed, genpack)
            if changed_refs:
                new_refs = {}
                for ref, sha in changed_refs.iteritems():
                    self.ui.status("    "+ remote_name + "::" + ref + " => GIT:" + sha[0:8] + "\n")
                    new_refs[ref] = sha
                self.git.set_remote_refs(new_refs, remote_name)
                self.update_hg_bookmarks(remote_name)
        except:
            # TODO : remove try/except or do something useful here
            raise

    # TODO : for now, we'll just push all heads that match remote heads
    #        * we should have specified push, tracking branches and --all
    # takes a dict of refs:shas from the server and returns what should be
    # pushed up
    def get_changed_refs(self, refs):
        keys = refs.keys()

        changed = {}
        if not keys:
            return None

        # TODO : this is a huge hack
        if keys[0] == 'capabilities^{}': # nothing on the server yet - first push
            changed['refs/heads/master'] = self.git.ref('master')

        tags = self.git.get_tags()
        for tag, sha in tags.iteritems():
            tag_name = 'refs/tags/' + tag
            if tag_name not in refs:
                changed[tag_name] = sha

        for ref_name in keys:
            parts = ref_name.split('/')
            if parts[0] == 'refs': # strip off 'refs/heads'
                if parts[1] == 'heads':
                    head = "/".join([v for v in parts[2:]])
                    local_ref = self.git.ref(ref_name)
                    if local_ref:
                        if not local_ref == refs[ref_name]:
                            changed[ref_name] = local_ref
        
        # Also push any local branches not on the server yet
        for head in self.local_heads():
            if not head in refs:
                ref = self.git.ref(head)
                changed[head] = ref

        return changed

    # takes a list of shas the server wants and shas the server has
    # and generates a list of commit shas we need to push up
    def generate_pack_contents(self, want, have):
        graph_walker = SimpleFetchGraphWalker(want, self.git.get_parents)
        next = graph_walker.next()
        shas = set()
        while next:
            if next in have:
                graph_walker.ack(next)
            else:
                shas.add(next)
            next = graph_walker.next()
        
        seen = []
        
        # so now i have the shas, need to turn them into a list of
        # tuples (sha, path) for ALL the objects i'm sending
        # TODO : don't send blobs or trees they already have
        def get_objects(tree, path):
            changes = list()
            changes.append((tree, path))
            for (mode, name, sha) in tree.entries():
                if mode == 0160000: # TODO : properly handle submodules and document what 57344 means
                    continue
                if sha in seen:
                    continue
                    
                obj = self.git.get_object(sha)
                seen.append(sha)
                if isinstance (obj, Blob):
                    changes.append((obj, path + name))
                elif isinstance(obj, Tree):
                    changes.extend(get_objects(obj, path + name + '/'))
            return changes

        objects = []
        for commit_sha in shas:
            commit = self.git.commit(commit_sha)
            objects.append((commit, 'commit'))
            tree = self.git.get_object(commit.tree)
            objects.extend( get_objects(tree, '/') )

        return objects

    def fetch_pack(self, remote_name):
        git_url = self.remote_name_to_url(remote_name)
        client, path = self.get_transport_and_path(git_url)
        graphwalker = SimpleFetchGraphWalker(self.git.heads().values(), self.git.get_parents)
        f, commit = self.git.object_store.add_pack()
        try:
            determine_wants = self.git.object_store.determine_wants_all
            refs = client.fetch_pack(path, determine_wants, graphwalker, f.write, sys.stdout.write)
            f.close()
            commit()
            if refs:
                self.git.set_remote_refs(refs, remote_name)
            else:
                self.ui.status(_("nothing new on the server\n"))
            return refs
        except:
            f.close()
            raise

    # take refs just fetched, add local tags for all tags not in .hgtags
    def import_local_tags(self, refs):
        keys = refs.keys()
        if not keys:
            return None
        for k in keys[0:]:
            ref_name = k
            parts = k.split('/')
            if (parts[0] == 'refs' and parts[1] == 'tags'):
                ref_name = "/".join([v for v in parts[2:]])
                if ref_name[-3:] == '^{}':
                    ref_name = ref_name[:-3]
                if not ref_name in self.repo.tags():
                    obj = self.git.get_object(refs[k])
                    sha = None
                    if isinstance (obj, Commit): # lightweight
                        sha = self.map_hg_get(refs[k])
                    if isinstance (obj, Tag): # annotated
                        (obj_type, obj_sha) = obj.get_object()
                        obj = self.git.get_object(obj_sha)
                        if isinstance (obj, Commit):                
                            sha = self.map_hg_get(obj_sha)
                    if sha:
                        self.repo.tag(ref_name, hex_to_sha(sha), '', True, None, None)
                    
        
    def import_git_objects(self, remote_name=None, refs=None):
        self.ui.status(_("importing Git objects into Hg\n"))
        # import heads and fetched tags as remote references
        todo = []
        done = set()
        convert_list = {}
        self.renames = {}

        # get a list of all the head shas
        if refs: 
          for head, sha in refs.iteritems():
            todo.append(sha)
        else:
          if remote_name:
              todo = self.git.remote_refs(remote_name).values()[:]
          elif self.importbranch:
              branches = self.importbranch.split(',')
              todo = [self.git.ref(i.strip()) for i in branches]
          else:
              todo = self.git.heads().values()[:]

        # traverse the heads getting a list of all the unique commits
        while todo:
            sha = todo.pop()
            assert isinstance(sha, str)
            if sha in done:
                continue
            done.add(sha)
            obj = self.git.get_object(sha)
            if isinstance (obj, Commit):                
                convert_list[sha] = obj
                todo.extend([p for p in obj.parents if p not in done])
            if isinstance(obj, Tag):
                (obj_type, obj_sha) = obj.get_object()
                obj = self.git.get_object(obj_sha)
                if isinstance (obj, Commit):                
                    convert_list[sha] = obj
                    todo.extend([p for p in obj.parents if p not in done])

        # sort the commits
        commits = toposort.TopoSort(convert_list).items()
        
        # import each of the commits, oldest first
        total = len(commits)
        magnitude = int(math.log(total, 10)) + 1 if total else 1
        for i, csha in enumerate(commits):
            if i%100 == 0:
                self.ui.status(_("at: %*d/%d\n") % (magnitude, i, total))
            commit = convert_list[csha]
            if not self.map_hg_get(csha): # it's already here
                self.import_git_commit(commit)
            else:
                # we need to get rename info for further upstream
                self.pseudo_import_git_commit(commit)

        self.update_hg_bookmarks(remote_name)

    def update_hg_bookmarks(self, remote_name):
        try:
            bms = bookmarks.parse(self.repo)
            if remote_name:
                heads = self.git.remote_refs(remote_name)
            else:
                branches = self.bookbranch.split(',')
                heads = dict((i, self.git.ref(i.strip())) for i in branches)

            base_name = (remote_name + '/') if remote_name else '' 

            for head, sha in heads.iteritems():
                if not sha:
                    self.ui.warn(_("Could not resolve head %s.\n") % head)
                    continue
                hgsha = hex_to_sha(self.map_hg_get(sha))
                if not head == 'HEAD':
                    bms[base_name + head] = hgsha
            if heads:
                bookmarks.write(self.repo, bms)

        except AttributeError:
            self.ui.warn(_('creating bookmarks failed, do you have'
                         ' bookmarks enabled?\n'))

    def convert_git_int_mode(self, mode):
	# TODO : make these into constants
        convert = {
         0100644: '',
         0100755: 'x',
         0120000: 'l'}
        if mode in convert:
            return convert[mode]
        return ''

    def extract_hg_metadata(self, message):
        split = message.split("\n\n--HG--\n", 1)
        renames = {}
        extra = {}
        files = []
        branch = False
        if len(split) == 2:
            message, meta = split
            lines = meta.split("\n")
            for line in lines:
                if line == '':
                    continue 
                
                command, data = line.split(" : ", 1)
                
                if command == 'rename':
                    before, after = data.split(" => ", 1)
                    renames[after] = before
                if command == 'branch':
                    branch = data
                if command == 'files':
                    files.append(data)
                if command == 'extra':
                    before, after = data.split(" : ", 1)
                    extra[before] = urllib.unquote(after)
        return (message, renames, branch, files, extra)

    def pseudo_import_git_commit(self, commit):
        (strip_message, hg_renames, hg_branch) = self.extract_hg_metadata(commit.message)
        cs = self.map_hg_get(commit.id)
        p1 = nullid
        p2 = nullid
        if len(commit.parents) > 0:
            sha = commit.parents[0]
            p1 = self.map_hg_get(sha)
        if len(commit.parents) > 1:
            sha = commit.parents[1]
            p2 = self.map_hg_get(sha)
        if len(commit.parents) > 2:
            # TODO : map extra parents to the extras file
            pass
        # saving rename info
        if (not (p2 == nullid) or (p1 == nullid)):
            self.renames[cs] = {}
        else:
            self.renames[cs] = self.renames[p1].copy()

        self.renames[cs].update(hg_renames)
    
    def import_git_commit(self, commit):
        self.ui.debug(_("importing: %s\n") % commit.id)
        # TODO : Do something less coarse-grained than try/except on the
        #        get_file call for removed files

        (strip_message, hg_renames, hg_branch, files, extra) = self.extract_hg_metadata(commit.message)
        
        # get a list of the changed, added, removed files
        files = self.git.get_files_changed(commit)

        date = (commit.author_time, -commit.author_timezone)
        text = strip_message

        def getfilectx(repo, memctx, f):
            try:
                (mode, sha, data) = self.git.get_file(commit, f)
                e = self.convert_git_int_mode(mode)
            except TypeError:
                raise IOError()
            if f in hg_renames:
                copied_path = hg_renames[f]
            else:
                copied_path = None
            return context.memfilectx(f, data, 'l' in e, 'x' in e, copied_path)

        gparents = map(self.map_hg_get, commit.parents)
        p1, p2 = (nullid, nullid)
        octopus = False

        if len(gparents) > 1:
            # merge, possibly octopus
            def commit_octopus(p1, p2):
                ctx = context.memctx(self.repo, (p1, p2), text, files, getfilectx,
                                     commit.author, date, {'hg-git': 'octopus'})
                return hex(self.repo.commitctx(ctx))

            octopus = len(gparents) > 2
            p2 = gparents.pop()
            p1 = gparents.pop()
            while len(gparents) > 0:
                p2 = commit_octopus(p1, p2)
                p1 = gparents.pop()
        else:
            if gparents:
                p1 = gparents.pop()

        # wierd hack for explicit file renames in first but not second branch
        if not (p2 == nullid):
            vals = [item for item in self.renames[p1].values() if not item in self.renames[p2].values()]
            for removefile in vals:
                files.remove(removefile)
        author = commit.author

        extra = {}
        if ' <none@none>' in commit.author:
            author = commit.author[:-12]

        # if named branch, add to extra
        if hg_branch:
            extra['branch'] = hg_branch

        # if committer is different than author, add it to extra
        if not commit._author_raw == commit._committer_raw:
            extra['committer'] = "%s %d %d" % (commit.committer, commit.commit_time, -commit.commit_timezone)

        if commit._encoding:
            extra['encoding'] = commit._encoding

        if hg_branch:
            extra['branch'] = hg_branch

        if octopus:
            extra['hg-git'] ='octopus-done'

        ctx = context.memctx(self.repo, (p1, p2), text, files, getfilectx,
                             author, date, extra)
        node = self.repo.commitctx(ctx)

        # save changeset to mapping file
        cs = hex(node)
        self.map_set(commit.id, cs)
        
        # saving rename info
        if (not (p2 == nullid) or (p1 == nullid)):
            self.renames[cs] = {}
        else:
            self.renames[cs] = self.renames[p1].copy()
            
        self.renames[cs].update(hg_renames)
        

    def check_bookmarks(self):
        if self.ui.config('extensions', 'hgext.bookmarks') is not None:
            self.ui.warn("YOU NEED TO SETUP BOOKMARKS\n")

    def get_transport_and_path(self, uri):
        from dulwich.client import TCPGitClient, SSHGitClient, SubprocessGitClient
        for handler, transport in (("git://", TCPGitClient), ("git@", SSHGitClient), ("git+ssh://", SSHGitClient)):
            if uri.startswith(handler):
                if handler == 'git@':
                    host, path = uri[len(handler):].split(":", 1)
                    host = 'git@' + host
                else:
                    host, path = uri[len(handler):].split("/", 1)
                return transport(host), '/' + path
        # if its not git or git+ssh, try a local url..
        return SubprocessGitClient(), uri

    def clear(self):
        mapfile = self.repo.join(self.mapfile)
        if os.path.exists(self.gitdir):
            for root, dirs, files in os.walk(self.gitdir, topdown=False):
                for name in files:
                    os.remove(os.path.join(root, name))
                for name in dirs:
                    os.rmdir(os.path.join(root, name))
            os.rmdir(self.gitdir)
        if os.path.exists(mapfile):
            os.remove(mapfile)
Exemplo n.º 11
0
class GitStorage():
    def _ignoreFile(self, dirName, fileName):
        """
        used for the copTree stuff
        ``dirName``
            the working directory
        ``fileName``
            list of files inside the directory (dirName)
        """
        result = []
        for i in fileName:
            path = dirName + i
            if path not in fileToIgnore:
                result.append(path)
        return result

    def _commit(self, tree):
        """
        commit a tree used only by the init
        ``tree``
            tree to commit
        """
        commit = Commit()
        commit.tree = tree.id
        commit.encoding = "UTF-8"
        commit.committer = commit.author = 'debexpo <%s>' % (
            pylons.config['debexpo.email'])
        commit.commit_time = commit.author_time = int(time())
        tz = parse_timezone('-0200')[0]
        commit.commit_timezone = commit.author_timezone = tz
        commit.message = " "
        self.repo.object_store.add_object(tree)
        self.repo.object_store.add_object(commit)
        self.repo.refs["HEAD"] = commit.id
        log.debug('commiting')
        return commit.id

    def __init__(self, path):
        #creating the repository
        if os.path.isdir(path):
            log.debug("directory exist, taking it as a git repository")
            self.repo = Repo(path)
        else:
            log.debug("directory doesn't exist, creating")
            os.makedirs(path)
            log.debug("initiate the repo")
            self.repo = Repo.init(path)
            log.debug("adding an empty tree to the repository")
            self._commit(Tree())

    #only this function will be used on upload
    def change(self, files):
        """
        used to change  afile in the git storage can be called for the first upload we don't care
        ``files``
            a list of file to change
        """
        if len(files) == 0:
            log.debug("trying to change nothing will do... nothing")
        else:
            log.debug("this will change %i files" % (len(files)))
            for f in files:
                self.repo.stage(str(f))
            log.debug("stages dones")
            self.repo.do_commit(
                "this is so awesome that nobody will never see it",
                committer="same here <*****@*****.**>")

    def buildTreeDiff(self, dest, tree=None, originalTree=None):
        """
        creating files from the diff between 2 trees, it will be used in the code browser to get older version
        (walking on history)
        ``tree``
            the tree that you want to compare to
        ``dest``
            the destination folder to build sources
        ``originalTree``
            the original Tree, by default it's the last one

        by default it's retun the last changed files

        """
        if tree is None:
            head = self.repo.commit(
                self.repo.commit(self.repo.head()).parents[0])
            tree = self.repo.tree(head.tree)
        if originalTree is None:
            originalTree = self.repo.tree(
                self.repo.commit(self.repo.head()).tree)
        blobToBuild = []
        #getting blob that have changed
        for blob in self.repo.object_store.iter_tree_contents(tree.id):
            if blob not in originalTree:
                blobToBuild.append(blob)
                fileToIgnore.append(blob.path)
        repoLocation = os.path.join(str(self.repo).split("'")[1])
        #creating the folder with link to older files
        if os.path.exists(repoLocation + dest):
            log.warning("%s already exist, copy will not work")
        else:
            log.debug("copying files")
            shutil.copytree(repoLocation,
                            repoLocation + dest,
                            symlinks=True,
                            ignore=self._ignoreFile)
        for b in blobToBuild:
            fileDirectory = os.path.split(b.path)
            fileDirectory.pop()
            if not os.path.exists(
                    os.path.join(repoLocation + dest,
                                 os.path.join(fileDirectory))):
                os.makedirs(
                    os.path.join(repoLocation + dest,
                                 os.path.join(fileDirectory)))
            file = open(os.path.join(repoLocation + dest, b.path), 'w')
            file.write(self.repo.get_object(b.sha).as_raw_string())
            file.close()
        tree = None
        originalTree = None

    #get*
    def getLastTree(self):
        """
        return the last tree
        """
        return self.repo.tree(self.repo._commit(self.repo.head()).tree)

    def getAllTrees(self):
        """
        return trees
        """
        result = []
        commit = self.repo._commit(self.repo.head())
        for c in commit._get_parents():
            result.append(c.tree)
        return result

    def getOlderFileContent(self, file):
        """
        return the first file's content that changed from the file
        ``file``
            the file to work on
        """
        with open(file) as f:
            originalBlob = Blob.from_string("".join(f.readlines()))
        trees = self.getAllTrees()
        for t in trees:
            #parsing tree in order to find the tree where the file change
            if originalBlob not in t:
                tree = t
                break
                #tree must be existent, other way file is not correct
        if tree is None:
            log.error(
                "there is no tree that contain this blob this souldn't happen, other way this file does not appear to come from this package"
            )
        else:
            if self.repo._commit(self.repo.head()).tree == tree:
                olderTree = self.repo.commit(
                    self.repo.head())._get_parents()[0].tree
            else:
                for c in self.repo._commit(self.repo.head())._get_parents():
                    if c.tree == tree:
                        try:
                            olderTree = c.get_parents()[0]
                        except IndexError:
                            log.debug("file is the last version")
                            olderTree = tree
            if olderTree != tree:
                #we must check here the blob that contains the older file
                for b in self.repo.object_store.iter_tree_contents(
                        olderTree.id):
                    if originalBlob.path == b.path:
                        #older blob find! awesome, in the first loop we already test if they are the same
                        # that's why we can now return the content of the file
                        return self.repo.get_object(b.sha).as_raw_string()
        return ""

    def getOlderCommits(self):
        """
        return a list of all commits
        """
        return self.repo.commit(self.repo.head())._get_parents()