Example #1
0
    def handle(self):
        write = lambda x: self.proto.write_sideband(1, x)

        graph_walker = ProtocolGraphWalker(self, self.repo.object_store,
                                           self.repo.get_peeled)
        objects_iter = self.repo.fetch_objects(graph_walker.determine_wants,
                                               graph_walker,
                                               self.progress,
                                               get_tagged=self.get_tagged)

        # Do they want any objects?
        if len(objects_iter) == 0:
            return

        #self.progress("dul-daemon says what\n")
        self.progress("counting objects: %d, done.\n" % len(objects_iter))
        write_pack_data(ProtocolFile(None, write), objects_iter,
                        len(objects_iter))
        #self.progress("how was that, then?\n")
        size = self.backend.repository.info.size
        msg = _('Repository size: %s\n' % filesizeformat(size))
        logging.info(msg)
        self.progress(msg)
        # we are done
        self.proto.write("0000")
Example #2
0
    def handle(self):
        def determine_wants(heads):
            keys = heads.keys()
            if keys:
                self.proto.write_pkt_line("%s %s\x00%s\n" % ( heads[keys[0]], keys[0], self.capabilities()))
                for k in keys[1:]:
                    self.proto.write_pkt_line("%s %s\n" % (heads[k], k))

            # i'm done..
            self.proto.write("0000")

            # Now client will either send "0000", meaning that it doesnt want to pull.
            # or it will start sending want want want commands
            want = self.proto.read_pkt_line()
            if want == None:
                return []

            want, self.client_capabilities = extract_capabilities(want)

            want_revs = []
            while want and want[:4] == 'want':
                want_revs.append(want[5:45])
                want = self.proto.read_pkt_line()
            return want_revs

        progress = lambda x: self.proto.write_sideband(2, x)
        write = lambda x: self.proto.write_sideband(1, x)

        class ProtocolGraphWalker(object):

            def __init__(self, proto):
                self.proto = proto
                self._last_sha = None

            def ack(self, have_ref):
                self.proto.write_pkt_line("ACK %s continue\n" % have_ref)

            def next(self):
                have = self.proto.read_pkt_line()
                if have[:4] == 'have':
                    return have[5:45]

                #if have[:4] == 'done':
                #    return None

                if self._last_sha:
                    # Oddness: Git seems to resend the last ACK, without the "continue" statement
                    self.proto.write_pkt_line("ACK %s\n" % self._last_sha)

                # The exchange finishes with a NAK
                self.proto.write_pkt_line("NAK\n")

        graph_walker = ProtocolGraphWalker(self.proto)
        (num_objects, objects_iter) = self.backend.fetch_objects(determine_wants, graph_walker, progress)
        progress("dul-daemon says what\n")
        progress("counting objects: %d, done.\n" % len(objects))
        write_pack_data(ProtocolFile(None, write), objects_iter, num_objects)
        progress("how was that, then?\n")
        # we are done
        self.proto.write("0000")
Example #3
0
    def add_objects(self, objects):
        """Add a set of objects to this object store.

        :param objects: Iterable over objects, should support __len__.
        """
        if len(objects) == 0:
            # Don't bother writing an empty pack file
            return
        f, commit = self.add_pack()
        write_pack_data(f, objects, len(objects))
        commit()
    def add_objects(self, objects):
        """Add a set of objects to this object store.

        :param objects: Iterable over objects, should support __len__.
        """
        if len(objects) == 0:
            # Don't bother writing an empty pack file
            return
        f, commit = self.add_pack()
        write_pack_data(f, objects, len(objects))
        commit()
    def add_pack_data(self, count, pack_data, progress=None):
        """Add pack data to this object store.

        :param num_items: Number of items to add
        :param pack_data: Iterator over pack data tuples
        """
        if count == 0:
            # Don't bother writing an empty pack file
            return
        f, commit, abort = self.add_pack()
        try:
            write_pack_data(f, count, pack_data, progress)
        except BaseException:
            abort()
            raise
        else:
            return commit()
Example #6
0
    def add_pack_data(self, count, pack_data, progress=None):
        """Add pack data to this object store.

        :param num_items: Number of items to add
        :param pack_data: Iterator over pack data tuples
        """
        if count == 0:
            # Don't bother writing an empty pack file
            return
        f, commit, abort = self.add_pack()
        try:
            write_pack_data(f, count, pack_data, progress)
        except BaseException:
            abort()
            raise
        else:
            return commit()
Example #7
0
    def handle(self):
        write = lambda x: self.proto.write_sideband(1, x)

        graph_walker = ProtocolGraphWalker(self, self.repo.object_store, self.repo.get_peeled)
        objects_iter = self.repo.fetch_objects(
            graph_walker.determine_wants, graph_walker, self.progress, get_tagged=self.get_tagged
        )

        # Did the process short-circuit (e.g. in a stateless RPC call)? Note
        # that the client still expects a 0-object pack in most cases.
        if objects_iter is None:
            return

        self.progress("dul-daemon says what\n")
        self.progress("counting objects: %d, done.\n" % len(objects_iter))
        write_pack_data(ProtocolFile(None, write), objects_iter, len(objects_iter))
        self.progress("how was that, then?\n")
        # we are done
        self.proto.write("0000")
Example #8
0
    def test_send_pack_new_ref(self):
        self.rin.write(
            b'0064310ca9477129b8586fa2afc779c1f57cf64bba6c '
            b'refs/heads/master\x00 report-status delete-refs ofs-delta\n'
            b'0000000eunpack ok\n'
            b'0019ok refs/heads/blah12\n'
            b'0000')
        self.rin.seek(0)

        tree = Tree()
        commit = Commit()
        commit.tree = tree
        commit.parents = []
        commit.author = commit.committer = b'test user'
        commit.commit_time = commit.author_time = 1174773719
        commit.commit_timezone = commit.author_timezone = 0
        commit.encoding = b'UTF-8'
        commit.message = b'test message'

        def update_refs(refs):
            return {
                b'refs/heads/blah12': commit.id,
                b'refs/heads/master':
                b'310ca9477129b8586fa2afc779c1f57cf64bba6c'
            }

        def generate_pack_data(have, want, ofs_delta=False):
            return pack_objects_to_data([
                (commit, None),
                (tree, b''),
            ])

        f = BytesIO()
        write_pack_data(f, *generate_pack_data(None, None))
        self.client.send_pack(b'/', update_refs, generate_pack_data)
        self.assertIn(self.rout.getvalue(), [
            b'007f0000000000000000000000000000000000000000 ' + commit.id +
            b' refs/heads/blah12\x00report-status ofs-delta0000' +
            f.getvalue(), b'007f0000000000000000000000000000000000000000 ' +
            commit.id + b' refs/heads/blah12\x00ofs-delta report-status0000' +
            f.getvalue()
        ])
Example #9
0
    def handle(self):

        progress = lambda x: self.proto.write_sideband(2, x)
        write = lambda x: self.proto.write_sideband(1, x)

        graph_walker = ProtocolGraphWalker(self)
        objects_iter = self.backend.fetch_objects(
          graph_walker.determine_wants, graph_walker, progress)

        # Do they want any objects?
        if len(objects_iter) == 0:
            return

        progress("dul-daemon says what\n")
        progress("counting objects: %d, done.\n" % len(objects_iter))
        write_pack_data(ProtocolFile(None, write), objects_iter, 
                        len(objects_iter))
        progress("how was that, then?\n")
        # we are done
        self.proto.write("0000")
Example #10
0
    def test_send_pack_new_ref(self):
        self.rin.write(
            b'0064310ca9477129b8586fa2afc779c1f57cf64bba6c '
            b'refs/heads/master\x00 report-status delete-refs ofs-delta\n'
            b'0000000eunpack ok\n'
            b'0019ok refs/heads/blah12\n'
            b'0000')
        self.rin.seek(0)

        tree = Tree()
        commit = Commit()
        commit.tree = tree
        commit.parents = []
        commit.author = commit.committer = b'test user'
        commit.commit_time = commit.author_time = 1174773719
        commit.commit_timezone = commit.author_timezone = 0
        commit.encoding = b'UTF-8'
        commit.message = b'test message'

        def determine_wants(refs):
            return {
                b'refs/heads/blah12': commit.id,
                b'refs/heads/master':
                    b'310ca9477129b8586fa2afc779c1f57cf64bba6c'
            }

        def generate_pack_data(have, want, ofs_delta=False):
            return pack_objects_to_data([(commit, None), (tree, b''), ])

        f = BytesIO()
        write_pack_data(f, *generate_pack_data(None, None))
        self.client.send_pack(b'/', determine_wants, generate_pack_data)
        self.assertIn(
            self.rout.getvalue(),
            [b'007f0000000000000000000000000000000000000000 ' + commit.id +
             b' refs/heads/blah12\x00report-status ofs-delta0000' +
             f.getvalue(),
             b'007f0000000000000000000000000000000000000000 ' + commit.id +
             b' refs/heads/blah12\x00ofs-delta report-status0000' +
             f.getvalue()])
Example #11
0
    def send_pack(self, path, determine_wants, generate_pack_contents):
        """Upload a pack to a remote repository.

        :param path: Repository path
        :param generate_pack_contents: Function that can return the shas of the
            objects to upload.

        :raises SendPackError: if server rejects the pack data
        :raises UpdateRefsError: if the server supports report-status
                                 and rejects ref updates
        """
        proto, unused_can_read = self._connect('receive-pack', path)
        old_refs, server_capabilities = self.read_refs(proto)
        if 'report-status' not in server_capabilities:
            self._send_capabilities.remove('report-status')
        new_refs = determine_wants(old_refs)
        if not new_refs:
            proto.write_pkt_line(None)
            return {}
        want = []
        have = [x for x in old_refs.values() if not x == ZERO_SHA]
        sent_capabilities = False
        for refname in set(new_refs.keys() + old_refs.keys()):
            old_sha1 = old_refs.get(refname, ZERO_SHA)
            new_sha1 = new_refs.get(refname, ZERO_SHA)
            if old_sha1 != new_sha1:
                if sent_capabilities:
                    proto.write_pkt_line('%s %s %s' % (old_sha1, new_sha1,
                                                            refname))
                else:
                    proto.write_pkt_line(
                      '%s %s %s\0%s' % (old_sha1, new_sha1, refname,
                                        ' '.join(self._send_capabilities)))
                    sent_capabilities = True
            if new_sha1 not in have and new_sha1 != ZERO_SHA:
                want.append(new_sha1)
        proto.write_pkt_line(None)
        if not want:
            return new_refs
        objects = generate_pack_contents(have, want)
        entries, sha = write_pack_data(proto.write_file(), objects,
                                       len(objects))

        if 'report-status' in self._send_capabilities:
            self._parse_status_report(proto)
        # wait for EOF before returning
        data = proto.read()
        if data:
            raise SendPackError('Unexpected response %r' % data)
        return new_refs
Example #12
0
    def handle(self):
        write = lambda x: self.proto.write_sideband(1, x)

        graph_walker = ProtocolGraphWalker(self, self.repo.object_store,
            self.repo.get_peeled)
        objects_iter = self.repo.fetch_objects(
          graph_walker.determine_wants, graph_walker, self.progress,
          get_tagged=self.get_tagged)

        # Do they want any objects?
        if len(objects_iter) == 0:
            return

        #self.progress("dul-daemon says what\n")
        self.progress("counting objects: %d, done.\n" % len(objects_iter))
        write_pack_data(ProtocolFile(None, write), objects_iter,
                        len(objects_iter))
        #self.progress("how was that, then?\n")
        size = self.backend.repository.info.size
        msg = _('Repository size: %s\n' % filesizeformat(size))
        logging.info(msg)
        self.progress(msg)
        # we are done
        self.proto.write("0000")
Example #13
0
    def send_pack(self, path, determine_wants, generate_pack_contents):
        """Upload a pack to a remote repository.

        :param path: Repository path
        :param generate_pack_contents: Function that can return the shas of the 
            objects to upload.
        """
        old_refs, server_capabilities = self.read_refs()
        new_refs = determine_wants(old_refs)
        if not new_refs:
            self.proto.write_pkt_line(None)
            return {}
        want = []
        have = [x for x in old_refs.values() if not x == ZERO_SHA]
        sent_capabilities = False
        for refname in set(new_refs.keys() + old_refs.keys()):
            old_sha1 = old_refs.get(refname, ZERO_SHA)
            new_sha1 = new_refs.get(refname, ZERO_SHA)
            if old_sha1 != new_sha1:
                if sent_capabilities:
                    self.proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1, refname))
                else:
                    self.proto.write_pkt_line("%s %s %s\0%s" % (old_sha1, new_sha1, refname, self.capabilities()))
                    sent_capabilities = True
            if not new_sha1 in (have, ZERO_SHA):
                want.append(new_sha1)
        self.proto.write_pkt_line(None)
        if not want:
            return new_refs
        objects = generate_pack_contents(have, want)
        (entries, sha) = write_pack_data(self.proto.write_file(), objects, 
                                         len(objects))
        
        # read the final confirmation sha
        try:
            client_sha = self.proto.read_pkt_line()
        except HangupException:
            # for git-daemon versions before v1.6.6.1-26-g38a81b4, there is
            # nothing to read; catch this and hide from the user.
            pass
        else:
            if not client_sha in (None, "", sha):
                raise ChecksumMismatch(sha, client_sha)

        return new_refs
Example #14
0
    def send_pack(self, path, determine_wants, generate_pack_contents):
        """Upload a pack to a remote repository.

        :param path: Repository path
        :param generate_pack_contents: Function that can return the shas of the 
            objects to upload.
        """
        old_refs, server_capabilities = self.read_refs()
        new_refs = determine_wants(old_refs)
        if not new_refs:
            self.proto.write_pkt_line(None)
            return {}
        want = []
        have = [x for x in old_refs.values() if not x == "0" * 40]
        sent_capabilities = False
        for refname in set(new_refs.keys() + old_refs.keys()):
            old_sha1 = old_refs.get(refname, "0" * 40)
            new_sha1 = new_refs.get(refname, "0" * 40)
            if old_sha1 != new_sha1:
                if sent_capabilities:
                    self.proto.write_pkt_line("%s %s %s" %
                                              (old_sha1, new_sha1, refname))
                else:
                    self.proto.write_pkt_line(
                        "%s %s %s\0%s" %
                        (old_sha1, new_sha1, refname, self.capabilities()))
                    sent_capabilities = True
            if not new_sha1 in (have, "0" * 40):
                want.append(new_sha1)
        self.proto.write_pkt_line(None)
        if not want:
            return new_refs
        objects = generate_pack_contents(have, want)
        (entries, sha) = write_pack_data(self.proto.write_file(), objects,
                                         len(objects))

        # read the final confirmation sha
        client_sha = self.proto.read(20)
        if not client_sha in (None, "", sha):
            raise ChecksumMismatch(sha, client_sha)

        return new_refs
Example #15
0
    def send_pack(self, path, determine_wants, generate_pack_contents):
        """Upload a pack to a remote repository.

        :param path: Repository path
        :param generate_pack_contents: Function that can return the shas of the 
            objects to upload.
        """
        old_refs, server_capabilities = self.read_refs()
        new_refs = determine_wants(old_refs)
        if not new_refs:
            self.proto.write_pkt_line(None)
            return {}
        want = []
        have = [x for x in old_refs.values() if not x == "0" * 40]
        sent_capabilities = False
        for refname in set(new_refs.keys() + old_refs.keys()):
            old_sha1 = old_refs.get(refname, "0" * 40)
            new_sha1 = new_refs.get(refname, "0" * 40)
            if old_sha1 != new_sha1:
                if sent_capabilities:
                    self.proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1, refname))
                else:
                    self.proto.write_pkt_line("%s %s %s\0%s" % (old_sha1, new_sha1, refname, self.capabilities()))
                    sent_capabilities = True
            if not new_sha1 in (have, "0" * 40):
                want.append(new_sha1)
        self.proto.write_pkt_line(None)
        if not want:
            return new_refs
        objects = generate_pack_contents(have, want)
        (entries, sha) = write_pack_data(self.proto.write_file(), objects, 
                                         len(objects))
        
        # read the final confirmation sha
        client_sha = self.proto.read(20)
        if not client_sha in (None, "", sha):
            raise ChecksumMismatch(sha, client_sha)
            
        return new_refs
Example #16
0
 def test_add_thin_pack(self):
     o = DiskObjectStore(self.store_dir)
     f, commit = o.add_thin_pack()
     b = make_object(Blob, data="more yummy data")
     write_pack_data(f, [(b, None)], 1)
     commit()