Beispiel #1
0
    def getAttachmentChunked(self, page, filename, chunkSize=DEFAULT_CHUNK):
        digest, size = self.getAttachmentInfo(page, filename)

        dataDigest = md5obj()
        current = 0

        while True:
            data = self.request("ChunkedAttachFile", page, filename, "load",
                                current, current + chunkSize)
            data = str(data)
            if not data:
                break

            current += len(data)
            yield data, current, size

            dataDigest.update(data)
            if current > size:
                break

        if current != size:
            raise WikiFailure("data changed while reading")

        dataDigest = dataDigest.hexdigest()
        if dataDigest != digest:
            raise WikiFailure("data changed while reading")
Beispiel #2
0
    def getAttachmentChunked(self, page, filename, chunkSize=DEFAULT_CHUNK):
        digest, size = self.getAttachmentInfo(page, filename)

        dataDigest = md5obj()
        current = 0

        while True:
            data = self.request("ChunkedAttachFile", page, filename, "load",
                                current, current + chunkSize)
            data = str(data)
            if not data:
                break

            current += len(data)
            yield data, current, size

            dataDigest.update(data)
            if current > size:
                break

        if current != size:
            raise WikiFailure("data changed while reading")

        dataDigest = dataDigest.hexdigest()
        if dataDigest != digest:
            raise WikiFailure("data changed while reading")
Beispiel #3
0
    def putAttachmentChunked(self,
                             page,
                             filename,
                             seekableStream,
                             chunksPerCheck=10,
                             chunkSize=DEFAULT_CHUNK,
                             log=True):
        count = 0
        total = 0
        digests = list()
        offsets = dict()

        while True:
            data = seekableStream.read(chunkSize)
            if not data:
                break

            digest = md5obj(data).hexdigest()
            digests.append(digest)

            length = len(data)
            offsets[digest] = total, length
            total += length

        while True:
            missing = self.request("ChunkedAttachFile", page, filename,
                                   "reassembly", chunkSize, digests)
            if not missing:
                return

            done = total
            for digest in missing:
                offset, length = offsets[digest]
                done -= length

            yield done, total

            missing = random.sample(missing, min(len(missing), chunksPerCheck))
            for digest in missing:
                offset, length = offsets[digest]

                seekableStream.seek(offset)
                data = seekableStream.read(length)

                try:
                    self.putCacheFile(page, digest, data, overwrite=True)
                except WikiFailure, e:
                    if e.fault.faultString == 'No such method: PageCache.':
                        self.putAttachment(page,
                                           digest,
                                           data,
                                           overwrite=True,
                                           log=False)
                    else:
                        raise

                count += 1
                done += length
                yield done, total
Beispiel #4
0
    def putAttachmentChunked(self, page, filename, seekableStream,
                             chunksPerCheck=10, chunkSize=DEFAULT_CHUNK, log=True):
        count = 0
        total = 0
        digests = list()
        offsets = dict()

        while True:
            data = seekableStream.read(chunkSize)
            if not data:
                break

            digest = md5obj(data).hexdigest()
            digests.append(digest)

            length = len(data)
            offsets[digest] = total, length
            total += length

        while True:
            missing = self.request("ChunkedAttachFile", page, filename,
                                   "reassembly", chunkSize, digests)
            if not missing:
                return

            done = total
            for digest in missing:
                offset, length = offsets[digest]
                done -= length

            yield done, total

            missing = random.sample(missing, min(len(missing), chunksPerCheck))
            for digest in missing:
                offset, length = offsets[digest]

                seekableStream.seek(offset)
                data = seekableStream.read(length)

                try:
                    self.putCacheFile(page, digest, data, overwrite=True)
                except WikiFailure, e:
                    if e.fault.faultString == 'No such method: PageCache.':
                        self.putAttachment(page, digest, data, overwrite=True, log=False)
                    else:
                        raise

                count += 1
                done += length
                yield done, total