class CacheTestCase(unittest.TestCase): """Test the cache class.""" def setUp(self): from swiftnbd.cache import Cache self.cache = Cache(10) def tearDown(self): pass def test_get_miss(self): data = self.cache.get(1) self.assertEqual(self.cache.ref[1], 0) self.assertEqual(data, None) def test_get_hit(self): self.cache.set(1, "DATA1") self.cache.set(2, "DATA2") data = self.cache.get(1) self.assertEqual(data, "DATA1") # set + get = 2 refferences self.assertEqual(self.cache.ref[1], 2) data = self.cache.get(2) self.assertEqual(data, "DATA2") def test_set(self): self.cache.set(1, "1") self.assertEqual(self.cache.ref[1], 1) self.cache.set(1, "1") self.assertEqual(self.cache.ref[1], 2) def test_limit(self): for i in range(10): self.cache.set(i, "DATA%s" % i) self.assertEqual(len(self.cache.data), 10) for i in range(10): self.assertEqual(self.cache.ref[i], 1) for i in range(10): for j in range(i+1): self.cache.get(i) # 0 should be discarded self.cache.set(10, "DATA11") self.assertEqual(len(self.cache.data), 10) self.assertEqual(self.cache.ref[0], 0) self.assertTrue(0 not in self.cache.data) # 10 should be discarded self.cache.set(11, "DATA12") self.assertEqual(len(self.cache.data), 10) self.assertEqual(self.cache.ref[10], 0) self.assertTrue(10 not in self.cache.data)
class CacheTestCase(unittest.TestCase): """Test the cache class.""" def setUp(self): from swiftnbd.cache import Cache self.cache = Cache(10) def tearDown(self): pass def test_get_miss(self): data = self.cache.get(1) self.assertEqual(self.cache.ref[1], 0) self.assertEqual(data, None) def test_get_hit(self): self.cache.set(1, "DATA1") self.cache.set(2, "DATA2") data = self.cache.get(1) self.assertEqual(data, "DATA1") # set + get = 2 refferences self.assertEqual(self.cache.ref[1], 2) data = self.cache.get(2) self.assertEqual(data, "DATA2") def test_set(self): self.cache.set(1, "1") self.assertEqual(self.cache.ref[1], 1) self.cache.set(1, "1") self.assertEqual(self.cache.ref[1], 2) def test_limit(self): for i in range(10): self.cache.set(i, "DATA%s" % i) self.assertEqual(len(self.cache.data), 10) for i in range(10): self.assertEqual(self.cache.ref[i], 1) for i in range(10): for j in range(i + 1): self.cache.get(i) # 0 should be discarded self.cache.set(10, "DATA11") self.assertEqual(len(self.cache.data), 10) self.assertEqual(self.cache.ref[0], 0) self.assertTrue(0 not in self.cache.data) # 10 should be discarded self.cache.set(11, "DATA12") self.assertEqual(len(self.cache.data), 10) self.assertEqual(self.cache.ref[10], 0) self.assertTrue(10 not in self.cache.data)
class SwiftStorage(object): """ Manages a object-split file stored in OpenStack Object Storage (swift). May raise StorageError (IOError). """ def __init__(self, auth, container, object_size, objects, cache=None, read_only=False): self.container = container self.object_size = object_size self.objects = objects self.pos = 0 self.locked = False self.meta = dict() self.read_only = read_only self.bytes_in = 0 self.bytes_out = 0 self.cache = cache if self.cache is None: self.cache = Cache(1024**2 // self.object_size) self.cli = client.Connection(**auth) def __str__(self): return self.container def lock(self, client_id): """Set the storage as busy""" if self.locked: return try: headers, _ = self.cli.get_container(self.container) except (socket.error, client.ClientException) as ex: raise StorageError(errno.EIO, "Failed to lock: %s" % ex) self.meta = getMeta(headers) if self.meta.get('client'): raise StorageError(errno.EBUSY, "Already in use: %s" % self.meta['client']) self.meta['client'] = "%s@%i" % (client_id, time()) hdrs = setMeta(self.meta) try: self.cli.put_container(self.container, headers=hdrs) except (socket.error, client.ClientException) as ex: raise StorageError(errno.EIO, "Failed to lock: %s" % ex) self.locked = True def unlock(self): """Set the storage as free""" if not self.locked: return self.meta['last'] = self.meta.get('client') self.meta['client'] = '' hdrs = setMeta(self.meta) try: self.cli.put_container(self.container, headers=hdrs) except (socket.error, client.ClientException) as ex: raise StorageError(errno.EIO, "Failed to unlock: %s" % ex) self.locked = False def read(self, size): data = bytearray() _size = size while _size > 0: obj = self.fetch_object(self.object_num) if obj == b'': break if _size + self.object_pos >= self.object_size: data += obj[self.object_pos:] part_size = self.object_size - self.object_pos else: data += obj[self.object_pos:self.object_pos+_size] part_size = _size _size -= part_size self.seek(self.pos + part_size) return data def write(self, data): if self.read_only: raise StorageError(errno.EROFS, "Read only storage") _data = data[:] if self.object_pos != 0: # object-align the beginning of data obj = self.fetch_object(self.object_num) _data = obj[:self.object_pos] + _data self.seek(self.pos - self.object_pos) reminder = len(_data) % self.object_size if reminder != 0: # object-align the end of data obj = self.fetch_object(self.object_num + (len(_data) // self.object_size)) _data += obj[reminder:] assert len(_data) % self.object_size == 0, "Data not aligned!" offs = 0 object_num = self.object_num while offs < len(_data): self.put_object(object_num, _data[offs:offs+self.object_size]) offs += self.object_size object_num += 1 def tell(self): return self.pos @property def object_pos(self): # position in the object return self.pos % self.object_size @property def object_num(self): # object number based on the position return self.pos // self.object_size @property def size(self): return self.object_size * self.objects def flush(self): self.cache.flush() def object_name(self, object_num): return "disk.part/%08i" % object_num def fetch_object(self, object_num): if object_num >= self.objects: return b'' data = self.cache.get(object_num) if not data: object_name = self.object_name(object_num) try: _, data = self.cli.get_object(self.container, object_name) except socket.error as ex: raise StorageError(errno.EIO, ex) except client.ClientException as ex: if ex.http_status != 404: raise StorageError(errno.EIO, ex) return b'\0' * self.object_size if len(data) != self.object_size: raise StorageError(errno.EIO, "Invalid object size (%s), %s expected" % len(data), self.object_size ) self.bytes_in += self.object_size self.cache.set(object_num, data) return data def put_object(self, object_num, data): if object_num >= self.objects: raise StorageError(errno.ESPIPE, "Write offset out of bounds") object_name = self.object_name(object_num) try: etag = self.cli.put_object(self.container, object_name, data) except (socket.error, client.ClientException) as ex: raise StorageError(errno.EIO, ex) checksum = md5(data).hexdigest() etag = etag.lower() if etag != checksum: raise StorageError(errno.EAGAIN, "Block integrity error (object_num=%s)" % object_num) self.bytes_out += self.object_size self.cache.set(object_num, data) def seek(self, offset): if offset < 0 or offset > self.size: raise StorageError(errno.ESPIPE, "Offset out of bounds") self.pos = offset
class SwiftStorage(object): """ Manages a object-split file stored in OpenStack Object Storage (swift). May raise StorageError (IOError). """ def __init__(self, auth, container, object_size, objects, cache=None, read_only=False): self.container = container self.object_size = object_size self.objects = objects self.pos = 0 self.locked = False self.meta = dict() self.read_only = read_only self.bytes_in = 0 self.bytes_out = 0 self.cache = cache if self.cache is None: self.cache = Cache(1024**2 // self.object_size) self.cli = client.Connection(**auth) def __str__(self): return self.container def lock(self, client_id): """Set the storage as busy""" if self.locked: return try: headers, _ = self.cli.get_container(self.container) except (socket.error, client.ClientException) as ex: raise StorageError(errno.EIO, "Failed to lock: %s" % ex) self.meta = getMeta(headers) if self.meta.get('client'): raise StorageError(errno.EBUSY, "Already in use: %s" % self.meta['client']) self.meta['client'] = "%s@%i" % (client_id, time()) hdrs = setMeta(self.meta) try: self.cli.put_container(self.container, headers=hdrs) except (socket.error, client.ClientException) as ex: raise StorageError(errno.EIO, "Failed to lock: %s" % ex) self.locked = True def unlock(self): """Set the storage as free""" if not self.locked: return self.meta['last'] = self.meta.get('client') self.meta['client'] = '' hdrs = setMeta(self.meta) try: self.cli.put_container(self.container, headers=hdrs) except (socket.error, client.ClientException) as ex: raise StorageError(errno.EIO, "Failed to unlock: %s" % ex) self.locked = False def read(self, size): data = bytearray() _size = size while _size > 0: obj = self.fetch_object(self.object_num) if obj == b'': break if _size + self.object_pos >= self.object_size: data += obj[self.object_pos:] part_size = self.object_size - self.object_pos else: data += obj[self.object_pos:self.object_pos + _size] part_size = _size _size -= part_size self.seek(self.pos + part_size) return data def write(self, data): if self.read_only: raise StorageError(errno.EROFS, "Read only storage") _data = data[:] if self.object_pos != 0: # object-align the beginning of data obj = self.fetch_object(self.object_num) _data = obj[:self.object_pos] + _data self.seek(self.pos - self.object_pos) reminder = len(_data) % self.object_size if reminder != 0: # object-align the end of data obj = self.fetch_object(self.object_num + (len(_data) // self.object_size)) _data += obj[reminder:] assert len(_data) % self.object_size == 0, "Data not aligned!" offs = 0 object_num = self.object_num while offs < len(_data): self.put_object(object_num, _data[offs:offs + self.object_size]) offs += self.object_size object_num += 1 def tell(self): return self.pos @property def object_pos(self): # position in the object return self.pos % self.object_size @property def object_num(self): # object number based on the position return self.pos // self.object_size @property def size(self): return self.object_size * self.objects def flush(self): self.cache.flush() def object_name(self, object_num): return "disk.part/%08i" % object_num def fetch_object(self, object_num): if object_num >= self.objects: return b'' data = self.cache.get(object_num) if not data: object_name = self.object_name(object_num) try: _, data = self.cli.get_object(self.container, object_name) except socket.error as ex: raise StorageError(errno.EIO, ex) except client.ClientException as ex: if ex.http_status != 404: raise StorageError(errno.EIO, ex) return b'\0' * self.object_size if len(data) != self.object_size: raise StorageError( errno.EIO, "Invalid object size (%s), %s expected" % len(data), self.object_size) self.bytes_in += self.object_size self.cache.set(object_num, data) return data def put_object(self, object_num, data): if object_num >= self.objects: raise StorageError(errno.ESPIPE, "Write offset out of bounds") object_name = self.object_name(object_num) try: etag = self.cli.put_object(self.container, object_name, data) except (socket.error, client.ClientException) as ex: raise StorageError(errno.EIO, ex) checksum = md5(data).hexdigest() etag = etag.lower() if etag != checksum: raise StorageError( errno.EAGAIN, "Block integrity error (object_num=%s)" % object_num) self.bytes_out += self.object_size self.cache.set(object_num, data) def seek(self, offset): if offset < 0 or offset > self.size: raise StorageError(errno.ESPIPE, "Offset out of bounds") self.pos = offset