def recalculateCommitMD5(commit, bundle):
with DebugInfo('Recalculating commit md5') as d:
from histo.server.get import openDataCodes
from pclib import hashstream
hasher = hashstream('md5')
with openDataCodes(bundle, commit) as f:
from pclib import copystream
copystream(f, hasher)
result = hasher.digest()
from pclib import byteshex
d.result = byteshex.encode(result)
return result
def writeData(self):
with DebugInfo('Send data. Expect size: %d' % self.commit['Size']):
with openDataCodes(self.dataBundle, self.commit) as f:
with DebugInfo('Copy stream') as d:
from pclib import copystream
sendSize = copystream(f, self.stream, self.commit['Size'])
d.result = 'Send size: %d' % sendSize
assert sendSize == self.commit['Size']
def readStreamToDataBundle(self, stream, size):
maxCodeSize = self.config['MaxCodeSize']
transferedSize = 0
result = []
while transferedSize < size:
remainSize = size - transferedSize
copySize = min(remainSize, maxCodeSize)
result.append(self.currentCodeCount)
dataName = 'data-%08d' % self.currentCodeCount
with self.dataBundle.open(dataName, 'wb') as f:
from pclib import copystream
assert copystream(stream, f, copySize) == copySize
self.currentCodeCount += 1
transferedSize += copySize
return result
def download(commit, extractRoot = 'D:\\'):
with connect() as conn:
conn.writeObject('Get')
p = {'CommitID' : commit['CommitID']}
conn.writeObject(p)
size = conn.readObject()
fileName = '%04d-%s.rar' % (commit['CommitID'], commit['Name'])
import os
fileName = os.path.join(extractRoot, fileName)
if os.path.exists(fileName):
return False
with open(fileName, 'wb') as f:
from pclib import copystream
assert size == copystream(conn, f, size)
assert getFileMd5(fileName) == commit['MD5']
return True
for e in x:
if type(e) is tuple:
for f in range(e[0], e[1]+1):
yield f
else:
yield e
def findMaxContinuous(codes):
for i in range(len(codes)):
if codes[i] - codes[0] != i:
return i + 1
return len(codes)
def openCodesAsStreams(bundle, codes, formatter):
for e in Codes.walk(codes):
with bundle.open(formatter(e), 'rb') as f:
yield f
def openCodesAsSingleStream(bundle, codes, formatter):
from histo.server.joinstream import JoinStream
streams = openCodesAsStreams(bundle, codes, formatter)
return JoinStream(streams)
if __name__ == '__main__':
from histo.bundle import Local
bundle = Local('D:\\histo-server\\data\\cache')
stream = openCodesAsSingleStream(bundle, [0,1,2,3,4], 'data-{:08d}'.format)
from pclib import copystream
with open('D:\\output.rar', 'wb') as f:
copystream(stream, f)
result = io.BytesIO()
while readSize < limit:
if self.currentStream is None:
try:
self.currentStream = next(self.streams)
except StopIteration:
break
remainSize = limit - readSize
read = self.currentStream.read(remainSize)
if not read:
self.currentStream = None
result.write(read)
readSize += len(read)
return result.getvalue()
def close(self):
pass
def __enter__(self):
return self
def __exit__(self, *k):
self.close()
if __name__ == '__main__':
from pclib import copystream
with open('D:\\1.txt', 'rb') as a:
with open('D:\\2.txt', 'rb') as b:
with open('D:\\12.txt', 'wb') as c:
copystream(JoinStream(iter([a,b])), c, chunksize=2)
def runTask2(self, task):
with self.fastBundle.open(task, "rb") as f1:
with self.slowBundle.open(task, "wb") as f2:
from pclib import copystream
copystream(f1, f2)
def getFileMd5(fileName):
from pclib import copystream, hashstream
result = hashstream('md5')
with open(fileName, 'rb') as f:
copystream(f, result)
return result.digest()