def getAttachmentChunked(self, page, filename, chunkSize=DEFAULT_CHUNK):
digest, size = self.getAttachmentInfo(page, filename)
dataDigest = md5obj()
current = 0
while True:
data = self.request("ChunkedAttachFile", page, filename, "load",
current, current + chunkSize)
data = str(data)
if not data:
break
current += len(data)
yield data, current, size
dataDigest.update(data)
if current > size:
break
if current != size:
raise WikiFailure("data changed while reading")
dataDigest = dataDigest.hexdigest()
if dataDigest != digest:
raise WikiFailure("data changed while reading")
def getAttachmentChunked(self, page, filename, chunkSize=DEFAULT_CHUNK):
digest, size = self.getAttachmentInfo(page, filename)
dataDigest = md5obj()
current = 0
while True:
data = self.request("ChunkedAttachFile", page, filename, "load",
current, current + chunkSize)
data = str(data)
if not data:
break
current += len(data)
yield data, current, size
dataDigest.update(data)
if current > size:
break
if current != size:
raise WikiFailure("data changed while reading")
dataDigest = dataDigest.hexdigest()
if dataDigest != digest:
raise WikiFailure("data changed while reading")
def putAttachmentChunked(self,
page,
filename,
seekableStream,
chunksPerCheck=10,
chunkSize=DEFAULT_CHUNK,
log=True):
count = 0
total = 0
digests = list()
offsets = dict()
while True:
data = seekableStream.read(chunkSize)
if not data:
break
digest = md5obj(data).hexdigest()
digests.append(digest)
length = len(data)
offsets[digest] = total, length
total += length
while True:
missing = self.request("ChunkedAttachFile", page, filename,
"reassembly", chunkSize, digests)
if not missing:
return
done = total
for digest in missing:
offset, length = offsets[digest]
done -= length
yield done, total
missing = random.sample(missing, min(len(missing), chunksPerCheck))
for digest in missing:
offset, length = offsets[digest]
seekableStream.seek(offset)
data = seekableStream.read(length)
try:
self.putCacheFile(page, digest, data, overwrite=True)
except WikiFailure, e:
if e.fault.faultString == 'No such method: PageCache.':
self.putAttachment(page,
digest,
data,
overwrite=True,
log=False)
else:
raise
count += 1
done += length
yield done, total
def putAttachmentChunked(self, page, filename, seekableStream,
chunksPerCheck=10, chunkSize=DEFAULT_CHUNK, log=True):
count = 0
total = 0
digests = list()
offsets = dict()
while True:
data = seekableStream.read(chunkSize)
if not data:
break
digest = md5obj(data).hexdigest()
digests.append(digest)
length = len(data)
offsets[digest] = total, length
total += length
while True:
missing = self.request("ChunkedAttachFile", page, filename,
"reassembly", chunkSize, digests)
if not missing:
return
done = total
for digest in missing:
offset, length = offsets[digest]
done -= length
yield done, total
missing = random.sample(missing, min(len(missing), chunksPerCheck))
for digest in missing:
offset, length = offsets[digest]
seekableStream.seek(offset)
data = seekableStream.read(length)
try:
self.putCacheFile(page, digest, data, overwrite=True)
except WikiFailure, e:
if e.fault.faultString == 'No such method: PageCache.':
self.putAttachment(page, digest, data, overwrite=True, log=False)
else:
raise
count += 1
done += length
yield done, total
