Пример #1
0
def recalculateCommitMD5(commit, bundle):
    with DebugInfo('Recalculating commit md5') as d:
        from histo.server.get import openDataCodes
        from pclib import hashstream
        hasher = hashstream('md5')
        with openDataCodes(bundle, commit) as f:
            from pclib import copystream
            copystream(f, hasher)
        result = hasher.digest()
        from pclib import byteshex
        d.result = byteshex.encode(result)
        return result
Пример #2
0
 def writeData(self):
     with DebugInfo('Send data. Expect size: %d' % self.commit['Size']):
         with openDataCodes(self.dataBundle, self.commit) as f:
             with DebugInfo('Copy stream') as d:
                 from pclib import copystream
                 sendSize = copystream(f, self.stream, self.commit['Size'])
                 d.result = 'Send size: %d' % sendSize
                 assert sendSize == self.commit['Size']
Пример #3
0
 def readStreamToDataBundle(self, stream, size):
     maxCodeSize = self.config['MaxCodeSize']
     transferedSize = 0
     result = []
     while transferedSize < size:
         remainSize = size - transferedSize
         copySize = min(remainSize, maxCodeSize)
         result.append(self.currentCodeCount)
         dataName = 'data-%08d' % self.currentCodeCount
         with self.dataBundle.open(dataName, 'wb') as f:
             from pclib import copystream
             assert copystream(stream, f, copySize) == copySize
         self.currentCodeCount += 1
         transferedSize += copySize
     return result
Пример #4
0
def download(commit, extractRoot = 'D:\\'):
    with connect() as conn:
        conn.writeObject('Get')
        p = {'CommitID' : commit['CommitID']}
        conn.writeObject(p)
        size = conn.readObject()
        fileName = '%04d-%s.rar' % (commit['CommitID'], commit['Name'])
        import os
        fileName = os.path.join(extractRoot, fileName)
        if os.path.exists(fileName):
            return False
        with open(fileName, 'wb') as f:
            from pclib import copystream
            assert size == copystream(conn, f, size)
        assert getFileMd5(fileName) == commit['MD5']
    return True
Пример #5
0
        for e in x:
            if type(e) is tuple:
                for f in range(e[0], e[1]+1):
                    yield f
            else:
                yield e

def findMaxContinuous(codes):
    for i in range(len(codes)):
        if codes[i] - codes[0] != i:
            return i + 1
    return len(codes)

def openCodesAsStreams(bundle, codes, formatter):
    for e in Codes.walk(codes):
        with bundle.open(formatter(e), 'rb') as f:
            yield f
            
def openCodesAsSingleStream(bundle, codes, formatter):
    from histo.server.joinstream import JoinStream
    streams = openCodesAsStreams(bundle, codes, formatter)
    return JoinStream(streams)

if __name__ == '__main__':
    from histo.bundle import Local
    bundle = Local('D:\\histo-server\\data\\cache')
    stream = openCodesAsSingleStream(bundle, [0,1,2,3,4], 'data-{:08d}'.format)
    from pclib import copystream
    with open('D:\\output.rar', 'wb') as f:
        copystream(stream, f)
Пример #6
0
        result = io.BytesIO()
        while readSize < limit:
            if self.currentStream is None:
                try:
                    self.currentStream = next(self.streams)
                except StopIteration:
                    break
            remainSize = limit - readSize
            read = self.currentStream.read(remainSize)
            if not read:
                self.currentStream = None
            result.write(read)
            readSize += len(read)
        return result.getvalue()
    
    def close(self):
        pass
    
    def __enter__(self):
        return self
    
    def __exit__(self, *k):
        self.close()

if __name__ == '__main__':
    from pclib import copystream
    with open('D:\\1.txt', 'rb') as a:
        with open('D:\\2.txt', 'rb') as b:
            with open('D:\\12.txt', 'wb') as c:
                copystream(JoinStream(iter([a,b])), c, chunksize=2)
Пример #7
0
    def runTask2(self, task):
        with self.fastBundle.open(task, "rb") as f1:
            with self.slowBundle.open(task, "wb") as f2:
                from pclib import copystream

                copystream(f1, f2)
Пример #8
0
def getFileMd5(fileName):
    from pclib import copystream, hashstream
    result = hashstream('md5')
    with open(fileName, 'rb') as f:
        copystream(f, result)
    return result.digest()