class DiscoZipFile(ZipFile, object): def __init__(self): self.buffer = BytesIO() super(DiscoZipFile, self).__init__(self.buffer, 'w', ZIP_DEFLATED) def writepath(self, pathname, exclude=()): for file in files(pathname): name, ext = os.path.splitext(file) if ext not in exclude: self.write(file, file) def writemodule(self, module, arcname=None): if isinstance(module, basestring): module = __import__(module) self.write(getsourcefile(module), arcname=arcname) def writesource(self, object): self.writepath(getsourcefile(getmodule(object))) def dump(self, handle): handle.write(self.dumps()) def dumps(self): self.buffer.seek(0) return self.buffer.read()
def setUp(self): self.ddfs.push('disco:test:blobs', [(BytesIO(b'datablob'), 'blobdata')]) self.ddfs.push('disco:test:blobs', [(BytesIO(b'datablob2'), 'blobdata2')]) self.ddfs.push('disco:test:emptyblob', [(BytesIO(b''), 'empty')]) self.ddfs.tag('disco:test:tag', [['urls']]) self.ddfs.tag('disco:test:metatag', [['tag://disco:test:tag'], ['tag://disco:test:metatag']])
class DiscoZipFile(ZipFile, object): def __init__(self): self.buffer = BytesIO() super(DiscoZipFile, self).__init__(self.buffer, 'w', ZIP_DEFLATED) def writepath(self, pathname, exclude=()): for file in files(pathname): name, ext = os.path.splitext(file) if ext not in exclude: self.write(file, file) def writemodule(self, module, arcname=None): if isinstance(module, basestring): module = __import__(module) self.write(getsourcefile(module), arcname=arcname) def writesource(self, object): self.writepath(getsourcefile(getmodule(object))) def dump(self, handle): handle.write(self.dumps()) def dumps(self): self.buffer.seek(0) return self.buffer.read()
def read(self, size=-1): buf = BytesIO() while size: bytes = self._read_chunk(size if size > 0 else CHUNK_SIZE) if not bytes: break size -= len(bytes) buf.write(bytes) return buf.getvalue()
def test_push(self): self.ddfs.push('disco:test:blobs', [(BytesIO(b'blobdata'), 'blobdata')]) self.assert_(self.ddfs.exists('disco:test:blobs')) self.ddfs.push('tag://disco:test:blobs2', [(BytesIO(b'blobdata'), 'blobdata')]) self.assert_(self.ddfs.exists('disco:test:blobs2')) self.ddfs.delete('disco:test:blobs') self.assert_(not self.ddfs.exists('disco:test:blobs')) self.ddfs.delete('disco:test:blobs2') self.assert_(not self.ddfs.exists('disco:test:blobs2'))
def read(self, size=-1): buf = BytesIO() while size: bytes = self._read_chunk(size if size > 0 else CHUNK_SIZE) if not bytes: break size -= len(bytes) buf.write(bytes) return buf.getvalue()
def setUp(self): self.ddfs.push('disco:test:authrd', [(BytesIO(b'datablob'), 'blobdata')]) self.ddfs.push('disco:test:authwr', [(BytesIO(b'datablob'), 'blobdata')]) self.ddfs.push('disco:test:authempty', [(BytesIO(b'datablob'), 'blobdata')]) self.ddfs.setattr('disco:test:authrd', 'a', 'v') self.ddfs.setattr('disco:test:authwr', 'a', 'v') self.ddfs.setattr('disco:test:authrd', 'ddfs:read-token', 'rdr') self.ddfs.setattr('disco:test:authwr', 'ddfs:write-token', 'wtr') self.ddfs.setattr('disco:test:authempty', 'a', 'v') self.ddfs.setattr('disco:test:authempty', 'ddfs:read-token', '') self.ddfs.setattr('disco:test:authempty', 'ddfs:write-token', '')
class DiscoOutputStream_v1(object): def __init__(self, stream, version=1, compression_level=2, min_hunk_size=HUNK_SIZE, max_record_size=None): self.stream = stream self.version = version self.compression_level = compression_level self.max_record_size = max_record_size self.min_hunk_size = min_hunk_size self.size = 0 self.hunk_size = 0 self.hunk = BytesIO() def add(self, k, v): self.append((k, v)) def append(self, record): self.hunk_write(pickle_dumps(record, 1)) if self.hunk_size > self.min_hunk_size: self.flush() def close(self): if self.hunk_size: self.flush() self.flush() def flush(self): hunk = self.hunk.getvalue() checksum = crc32(hunk) & 0xFFFFFFFF iscompressed = int(self.compression_level > 0) if iscompressed: hunk = compress(hunk, self.compression_level) data = b''.join([struct.pack('<BBIQ', 128 + self.version, iscompressed, checksum, len(hunk)), hunk]) self.stream.write(data) self.size += len(data) self.hunk_size = 0 self.hunk = BytesIO() def hunk_write(self, data): size = len(data) if self.max_record_size and size > self.max_record_size: raise ValueError("Record of size " + str(size) + " is larger than max_record_size: " + str(self.max_record_size)) self.hunk.write(data) self.hunk_size += size
class DiscoOutputStream_v1(object): def __init__(self, stream, version=1, compression_level=2, min_hunk_size=HUNK_SIZE, max_record_size=None): self.stream = stream self.version = version self.compression_level = compression_level self.max_record_size = max_record_size self.min_hunk_size = min_hunk_size self.size = 0 self.hunk_size = 0 self.hunk = BytesIO() def add(self, k, v): self.append((k, v)) def append(self, record): self.hunk_write(pickle_dumps(record, 1)) if self.hunk_size > self.min_hunk_size: self.flush() def close(self): if self.hunk_size: self.flush() self.flush() def flush(self): hunk = self.hunk.getvalue() checksum = crc32(hunk) & 0xFFFFFFFF iscompressed = int(self.compression_level > 0) if iscompressed: hunk = compress(hunk, self.compression_level) data = b''.join([struct.pack('<BBIQ', 128 + self.version, iscompressed, checksum, len(hunk)), hunk]) self.stream.write(data) self.size += len(data) self.hunk_size = 0 self.hunk = BytesIO() def hunk_write(self, data): size = len(data) if self.max_record_size and size > self.max_record_size: raise ValueError("Record of size " + str(size) + " is larger than max_record_size: " + str(self.max_record_size)) self.hunk.write(data) self.hunk_size += size
def __init__(self, stream, version=1, compression_level=2, min_hunk_size=HUNK_SIZE, max_record_size=None): self.stream = stream self.version = version self.compression_level = compression_level self.max_record_size = max_record_size self.min_hunk_size = min_hunk_size self.size = 0 self.hunk_size = 0 self.hunk = BytesIO()
def flush(self): hunk = self.hunk.getvalue() checksum = crc32(hunk) & 0xFFFFFFFF iscompressed = int(self.compression_level > 0) if iscompressed: hunk = compress(hunk, self.compression_level) data = b''.join([ struct.pack('<BBIQ', 128 + self.version, iscompressed, checksum, len(hunk)), hunk ]) self.stream.write(data) self.size += len(data) self.hunk_size = 0 self.hunk = BytesIO()
def jobpack(self, jobname): """Return the :class:`disco.job.JobPack` submitted for the job.""" from disco.compat import BytesIO from disco.job import JobPack return JobPack.load( BytesIO( self.request('/disco/ctrl/parameters?name={0}'.format(jobname), as_bytes=True)))
def test_create_delete_create_token(self): self.ddfs.delete('disco:test:delete2') self.assert_(not self.ddfs.exists('disco:test:delete2')) self.ddfs.push('disco:test:delete2', [(BytesIO(b'abc'), 'atom')], token='secret1') self.assert_(self.ddfs.exists('disco:test:delete2')) self.assert_("disco:test:delete2" in self.ddfs.list('disco:test:delete2'))
def test_create_delete_create(self): self.ddfs.delete('disco:test:delete1') self.assert_(not self.ddfs.exists('disco:test:delete1')) self.ddfs.push('disco:test:delete1', [(BytesIO(b'datablob'), 'blobdata')]) self.assert_(self.ddfs.exists('disco:test:delete1')) self.assert_( "disco:test:delete1" in self.ddfs.list('disco:test:delete1'))
def __iter__(self): chunk = self._read_chunk(CHUNK_SIZE) while chunk: next_chunk = self._read_chunk(CHUNK_SIZE) lines = list(BytesIO(chunk)) last = lines.pop() if next_chunk else b'' for line in lines: yield line chunk = last + next_chunk
def __init__(self, stream, version=1, compression_level=2, min_hunk_size=HUNK_SIZE, max_record_size=None): self.stream = stream self.version = version self.compression_level = compression_level self.max_record_size = max_record_size self.min_hunk_size = min_hunk_size self.size = 0 self.hunk_size = 0 self.hunk = BytesIO()
def flush(self): hunk = self.hunk.getvalue() checksum = crc32(hunk) & 0xFFFFFFFF iscompressed = int(self.compression_level > 0) if iscompressed: hunk = compress(hunk, self.compression_level) data = b''.join([struct.pack('<BBIQ', 128 + self.version, iscompressed, checksum, len(hunk)), hunk]) self.stream.write(data) self.size += len(data) self.hunk_size = 0 self.hunk = BytesIO()
def disco_input_stream(stream, size, url, ignore_corrupt=False): """Input stream for Disco's internal compression format.""" from disco.compat import BytesIO, int_of_byte from disco.compat import pickle_load import struct, gzip, zlib offset = 0 while True: header = stream.read(1) if not header: return if int_of_byte(header[0]) < 128: for e in old_netstr_reader(stream, size, url, header): yield e return try: is_compressed, checksum, hunk_size =\ struct.unpack('<BIQ', stream.read(13)) except: raise DataError("Truncated data at {0} bytes".format(offset), url) if not hunk_size: return hunk = stream.read(hunk_size) data = b'' try: data = zlib.decompress(hunk) if is_compressed else hunk if checksum != (zlib.crc32(data) & 0xFFFFFFFF): raise ValueError("Checksum does not match") except (ValueError, zlib.error) as e: if not ignore_corrupt: raise DataError( "Corrupted data between bytes {0}-{1}: {2}".format( offset, offset + hunk_size, e), url) offset += hunk_size hunk = BytesIO(data) while True: try: yield pickle_load(hunk) except EOFError: break except UnpicklingError as e: if not ignore_corrupt: raise DataError( "Corrupted data between bytes {0}-{1}: {2}".format( offset - hunk_size, offset, e), url)
def chunk(self, tag, urls, replicas=None, forceon=[], retries=10, delayed=False, update=False, token=None, chunk_size=CHUNK_SIZE, max_record_size=MAX_RECORD_SIZE, **kwargs): """ Chunks the contents of `urls`, pushes the chunks to ddfs and tags them with `tag`. """ from disco.core import result_iterator if 'reader' not in kwargs: kwargs['reader'] = None def chunk_iter(replicas): chunker = Chunker(chunk_size=chunk_size, max_record_size=max_record_size) return chunker.chunks(result_iterator([replicas], **kwargs)) def chunk_name(replicas, n): url = listify(replicas)[0] return self.safe_name('{0}-{1}'.format(os.path.basename(url), n)) blobs = [ self._push((BytesIO(chunk), chunk_name(reps, n)), replicas=replicas, forceon=forceon, retries=retries) for reps in urls for n, chunk in enumerate(chunk_iter(reps)) ] return (self.tag(tag, blobs, delayed=delayed, update=update, token=token), blobs)
def test_atomic_token(self): self.ddfs.push('disco:test:atomic1', [(BytesIO(b'abc'), 'atom')], update=True, delayed=True, token='secret1') getter = lambda: self.ddfs.getattr('disco:test:atomic1', 'foobar') self.assertCommErrorCode(401, getter) self.assertEquals(self.ddfs.getattr('disco:test:atomic1', 'ddfs:write-token', token='secret1'), 'secret1') self.ddfs.put('disco:test:atomic2', [], token='secret2') getter = lambda: self.ddfs.getattr('disco:test:atomic2', 'foobar') self.assertCommErrorCode(401, getter) self.assertEquals(self.ddfs.getattr('disco:test:atomic2', 'ddfs:write-token', token='secret2'), 'secret2') self.ddfs.put('disco:test:notoken', []) self.assertEquals(self.ddfs.getattr('disco:test:notoken', 'ddfs:write-token'), None)
def read(self): if self.isopen: return BytesIO(str_to_bytes(self.source)).read return open(self.source, 'rb').read
def __init__(self): self.buffer = BytesIO() super(DiscoZipFile, self).__init__(self.buffer, 'w', ZIP_DEFLATED)
def codec(self, version=1, corrupt=False, ignore_corrupt=False, **kwargs): buf = BytesIO() stream = DiscoOutputStream(buf, version=version, **kwargs) t = self.encode(stream, self.data) final_size = len(buf.getvalue()) final_mb = final_size / 1024**2 msg = (("{0:1.2f}MB encoded in {1:1.3f}s ({2:1.2f}MB/s), " "encoded size {3:1.3f}MB (version: {4}, {5})") .format(self.size, t, self.size / t, final_mb, version, kwargs)) if corrupt: buf.seek(0) new = BytesIO() new.write(buf.read(100)) new.write(b'X') buf.read(1) new.write(buf.read()) buf = new buf.seek(0) t, res = self.decode(buf, final_size, "nourl", ignore_corrupt=ignore_corrupt) if not ignore_corrupt: print("{0}, decoded in {1:1.3f}s ({2:1.2f}MB/s)" .format(msg, t, self.size / t)) return res
def setUp(self): self.ddfs.push('disco:test:attrs', [(BytesIO(b'datablob'), 'blobdata')]) self.ddfs.setattr('disco:test:attrs', 'a1', 'v1') self.ddfs.setattr('disco:test:attrs', 'a2', 'v2')
def makeout(self): return DiscoOutputStream(BytesIO(), max_record_size=MAX_RECORD_SIZE)
def __init__(self): self.buffer = BytesIO() super(DiscoZipFile, self).__init__(self.buffer, 'w', ZIP_DEFLATED)
def setUp(self): self.ddfs.push('disco:test:delete1', [(BytesIO(b'datablob'), 'blobdata')]) self.ddfs.push('disco:test:delete2', [(BytesIO(b'datablob'), 'blobdata')])
def save_oob(host, name, key, value, ddfs_token=None): from disco.ddfs import DDFS DDFS(host).push(DDFS.job_oob(name), [(BytesIO(value), key)], delayed=True)
class DDFSUpdateTestCase(TestCase): data = BytesIO(b'blobdata') def setUp(self): self.ddfs.delete('disco:test:blobs') def blobnames(self, tag): from disco.ddfs import DDFS return list(reversed(list(DDFS.blob_name(repl[0]) for repl in self.ddfs.blobs(tag)))) def test_update_empty_new(self): self.ddfs.push('disco:test:blobs', [], update=True) self.assertEquals(len(self.blobnames('disco:test:blobs')), 0) self.ddfs.delete('disco:test:blobs') def test_update(self): for i in range(5): self.ddfs.push('disco:test:blobs', [(self.data, 'dup')] * 2, update=True) self.assertEquals(len(self.blobnames('disco:test:blobs')), 1) for i in range(5): self.ddfs.push('disco:test:blobs', [(self.data, 'dup2')], update=True, delayed=True) self.assertEquals(len(self.blobnames('disco:test:blobs')), 2) self.ddfs.delete('disco:test:blobs') def test_no_garbage(self): self.ddfs.push('disco:test:blobs', [(self.data, 'dup')] * 2, update=True) tag_pre = self.ddfs.get('disco:test:blobs') self.assertEquals(len(tag_pre['urls']), 1) self.ddfs.tag('disco:test:blobs', tag_pre['urls'], update=True) self.assertEquals(tag_pre['id'], self.ddfs.get('disco:test:blobs')['id']) self.ddfs.delete('disco:test:blobs') def test_random(self): import random keys = [str(random.randint(1, 100)) for i in range(100)] ukeys = [] for key in keys: self.ddfs.push('disco:test:blobs', [(self.data, key)], update=True) if key not in ukeys: ukeys.append(key) self.assertEquals(ukeys, self.blobnames('disco:test:blobs')) self.ddfs.delete('disco:test:blobs') def test_mixed(self): keys = [] for key in map(str, range(10)): self.ddfs.push('disco:test:blobs', [(self.data, key)] * 2) keys += [key] * 2 for key in map(str, range(15)): self.ddfs.push('disco:test:blobs', [(self.data, key)] * 2, update=True) if int(key) > 9: keys.append(key) for key in map(str, range(10)): self.ddfs.push('disco:test:blobs', [(self.data, key)] * 2, delayed=True) keys += [key] * 2 self.assertEquals(keys, self.blobnames('disco:test:blobs')) self.ddfs.delete('disco:test:blobs') def tearDown(self): self.ddfs.delete('disco:test:blobs')
def dumps(obj, protocol=None): file = BytesIO() Pickler(file, protocol).dump(obj) return file.getvalue()
def setUp(self): super(AuthTestCase, self).setUp() from disco.compat import BytesIO self.tag = 'disco:test:authjob' self.ddfs.push(self.tag, [(BytesIO(b'blobdata'), 'blob')])
def makeout(self): return DiscoOutputStream(BytesIO(), max_record_size=self.max_record_size)