Exemplo n.º 1
0
    def _fetch_pack(self):
        """Fetch changes and store them in a pack."""

        def prepare_refs(refs):
            return [ref.hash.encode('utf-8') for ref in refs
                    if not ref.refname.endswith('^{}')]

        def determine_wants(refs):
            remote_refs = prepare_refs(self._discover_refs(remote=True))
            local_refs = prepare_refs(self._discover_refs())
            wants = [ref for ref in remote_refs if ref not in local_refs]
            return wants

        client, repo_path = dulwich.client.get_transport_and_path(self.uri)
        repo = dulwich.repo.Repo(self.dirpath)
        fd = io.BytesIO()

        local_refs = self._discover_refs()
        graph_walker = _GraphWalker(local_refs)

        result = client.fetch_pack(repo_path,
                                   determine_wants,
                                   graph_walker,
                                   fd.write)
        refs = [GitRef(ref_hash.decode('utf-8'), ref_name.decode('utf-8'))
                for ref_name, ref_hash in result.refs.items()]

        if len(fd.getvalue()) > 0:
            fd.seek(0)
            pack = repo.object_store.add_thin_pack(fd.read, None)
            pack_name = pack.name().decode('utf-8')
        else:
            pack_name = None

        return (pack_name, refs)
Exemplo n.º 2
0
 def fetch_pack(self, remote_name, heads):
     client, path = self.get_transport_and_path(remote_name)
     graphwalker = self.git.get_graph_walker()
     def determine_wants(refs):
         if heads:
             want = []
             # contains pairs of ('refs/(heads|tags|...)/foo', 'foo')
             # if ref is just '<foo>', then we get ('foo', 'foo')
             stripped_refs = [
                 (r, r[r.find('/', r.find('/')+1)+1:])
                     for r in refs]
             for h in heads:
                 r = [pair[0] for pair in stripped_refs if pair[1] == h]
                 if not r:
                     raise hgutil.Abort("ref %s not found on remote server" % h)
                 elif len(r) == 1:
                     want.append(refs[r[0]])
                 else:
                     raise hgutil.Abort("ambiguous reference %s: %r" % (h, r))
         else:
             want = [sha for ref, sha in refs.iteritems()
                     if not ref.endswith('^{}')]
         return want
     f, commit = self.git.object_store.add_pack()
     try:
         try:
             return client.fetch_pack(path, determine_wants, graphwalker,
                                      f.write, self.ui.status)
         except HangupException:
             raise hgutil.Abort("the remote end hung up unexpectedly")
     finally:
         commit()
Exemplo n.º 3
0
 def fetch_pack(self, remote_name, heads):
     client, path = self.get_transport_and_path(remote_name)
     graphwalker = self.git.get_graph_walker()
     def determine_wants(refs):
         if heads:
             want = []
             for h in heads:
                 r = [ref for ref in refs if ref.endswith('/'+h)]
                 if not r:
                     raise hgutil.Abort("ref %s not found on remote server")
                 elif len(r) == 1:
                     want.append(refs[r[0]])
                 else:
                     raise hgutil.Abort("ambiguous reference %s: %r"%(h, r))
         else:
             want = [sha for ref, sha in refs.iteritems()
                     if not ref.endswith('^{}')]
         return want
     f, commit = self.git.object_store.add_pack()
     try:
         try:
             return client.fetch_pack(path, determine_wants, graphwalker, f.write, self.ui.status)
         except HangupException:
             raise hgutil.Abort("the remote end hung up unexpectedly")
     finally:
         commit()
Exemplo n.º 4
0
def list_remote_refs(git_url):
    """Get the refs from a remote git repo as a dictionary of name->hash."""
    client, path = dulwich.client.get_transport_and_path(git_url)
    try:
        return client.fetch_pack(path, lambda refs: [], None, None)
    except dulwich.errors.HangupException as e:
        raise LSRemoteException("Unable to fetch remote refs: %s" % e)
Exemplo n.º 5
0
def list_remote_refs(git_url):
    """Get the refs from a remote git repo as a dictionary of name->hash."""
    client, path = dulwich.client.get_transport_and_path(git_url)
    try:
        refs = client.fetch_pack(path, lambda refs: [], None, lambda data: None)
        return {k.decode('UTF-8'): v.decode('UTF-8') for k, v in refs.items()}
    except dulwich.errors.HangupException as e:
        raise LSRemoteException("Unable to fetch remote refs: %s" % e)
Exemplo n.º 6
0
    def fetch_pack(self, remote_name, heads):
        client, path = self.get_transport_and_path(remote_name)
        graphwalker = self.git.get_graph_walker()

        def determine_wants(refs):
            if heads:
                want = []
                # contains pairs of ('refs/(heads|tags|...)/foo', 'foo')
                # if ref is just '<foo>', then we get ('foo', 'foo')
                stripped_refs = [(r, r[r.find('/',
                                              r.find('/') + 1) + 1:])
                                 for r in refs]
                for h in heads:
                    r = [pair[0] for pair in stripped_refs if pair[1] == h]
                    if not r:
                        raise hgutil.Abort(
                            "ref %s not found on remote server" % h)
                    elif len(r) == 1:
                        want.append(refs[r[0]])
                    else:
                        raise hgutil.Abort(
                            "ambiguous reference %s: %r" % (h, r))
            else:
                want = [
                    sha for ref, sha in refs.iteritems()
                    if not ref.endswith('^{}') and (ref.startswith(
                        'refs/heads/') or ref.startswith('refs/tags/'))
                ]
            want = [x for x in want if x not in self.git]
            return want

        f, commit = self.git.object_store.add_pack()
        try:
            try:
                progress = GitProgress(self.ui)
                ret = client.fetch_pack(path, determine_wants, graphwalker,
                                        f.write, progress.progress)
                progress.flush()
                return ret
            except (HangupException, GitProtocolError), e:
                raise hgutil.Abort(_("git remote error: ") + str(e))
        finally:
            commit()
Exemplo n.º 7
0
    def fetch_pack(self, remote_name, heads):
        client, path = self.get_transport_and_path(remote_name)
        graphwalker = self.git.get_graph_walker()

        def determine_wants(refs):
            if heads:
                want = []
                # contains pairs of ('refs/(heads|tags|...)/foo', 'foo')
                # if ref is just '<foo>', then we get ('foo', 'foo')
                stripped_refs = [(r, r[r.find('/',
                                              r.find('/') + 1) + 1:])
                                 for r in refs]
                for h in heads:
                    r = [pair[0] for pair in stripped_refs if pair[1] == h]
                    if not r:
                        raise hgutil.Abort(
                            "ref %s not found on remote server" % h)
                    elif len(r) == 1:
                        want.append(refs[r[0]])
                    else:
                        raise hgutil.Abort("ambiguous reference %s: %r" %
                                           (h, r))
            else:
                want = [
                    sha for ref, sha in refs.iteritems()
                    if not ref.endswith('^{}') and (ref.startswith(
                        'refs/heads/') or ref.startswith('refs/tags/'))
                ]
            want = [x for x in want if x not in self.git]
            return want

        f, commit = self.git.object_store.add_pack()
        try:
            try:
                progress = GitProgress(self.ui)
                ret = client.fetch_pack(path, determine_wants, graphwalker,
                                        f.write, progress.progress)
                progress.flush()
                return ret
            except (HangupException, GitProtocolError), e:
                raise hgutil.Abort(_("git remote error: ") + str(e))
        finally:
            commit()
Exemplo n.º 8
0
    def fetch_pack(self, remote_name, heads):
        client, path = self.get_transport_and_path(remote_name)
        graphwalker = self.git.get_graph_walker()

        def determine_wants(refs):
            if heads:
                want = []
                # contains pairs of ('refs/(heads|tags|...)/foo', 'foo')
                # if ref is just '<foo>', then we get ('foo', 'foo')
                stripped_refs = [(r, r[r.find('/',
                                              r.find('/') + 1) + 1:])
                                 for r in refs]
                for h in heads:
                    r = [pair[0] for pair in stripped_refs if pair[1] == h]
                    if not r:
                        raise hgutil.Abort(
                            "ref %s not found on remote server" % h)
                    elif len(r) == 1:
                        want.append(refs[r[0]])
                    else:
                        raise hgutil.Abort("ambiguous reference %s: %r" %
                                           (h, r))
            else:
                want = [
                    sha for ref, sha in refs.iteritems()
                    if not ref.endswith('^{}')
                ]
            return want

        f, commit = self.git.object_store.add_pack()
        try:
            try:
                return client.fetch_pack(path, determine_wants, graphwalker,
                                         f.write, self.ui.status)
            except HangupException:
                raise hgutil.Abort("the remote end hung up unexpectedly")
        finally:
            commit()
Exemplo n.º 9
0
    def fetch_pack_from_origin(
        self,
        origin_url: str,
        base_repo: RepoRepresentation,
        do_activity: Callable[[bytes], None],
    ) -> FetchPackReturn:
        """Fetch a pack from the origin"""

        pack_buffer = SpooledTemporaryFile(max_size=self.temp_file_cutoff)
        transport_url = origin_url

        logger.debug("Transport url to communicate with server: %s", transport_url)

        client, path = dulwich.client.get_transport_and_path(
            transport_url, thin_packs=False
        )

        logger.debug("Client %s to fetch pack at %s", client, path)

        size_limit = self.pack_size_bytes

        def do_pack(data: bytes) -> None:
            cur_size = pack_buffer.tell()
            would_write = len(data)
            if cur_size + would_write > size_limit:
                raise IOError(
                    f"Pack file too big for repository {origin_url}, "
                    f"limit is {size_limit} bytes, current size is {cur_size}, "
                    f"would write {would_write}"
                )

            pack_buffer.write(data)

        pack_result = client.fetch_pack(
            path,
            base_repo.determine_wants,
            base_repo.graph_walker(),
            do_pack,
            progress=do_activity,
        )

        remote_refs = pack_result.refs or {}
        symbolic_refs = pack_result.symrefs or {}

        pack_buffer.flush()
        pack_size = pack_buffer.tell()
        pack_buffer.seek(0)

        logger.debug("fetched_pack_size=%s", pack_size)

        # check if repository only supports git dumb transfer protocol,
        # fetched pack file will be empty in that case as dulwich do
        # not support it and do not fetch any refs
        self.dumb = transport_url.startswith("http") and getattr(client, "dumb", False)

        return FetchPackReturn(
            remote_refs=utils.filter_refs(remote_refs),
            symbolic_refs=utils.filter_refs(symbolic_refs),
            pack_buffer=pack_buffer,
            pack_size=pack_size,
        )