def __init__(self,
authurl,
username,
password,
container,
object_size,
objects,
cache=None,
read_only=False):
self.container = container
self.object_size = object_size
self.objects = objects
self.pos = 0
self.locked = False
self.meta = dict()
self.read_only = read_only
self.bytes_in = 0
self.bytes_out = 0
self.cache = cache
if self.cache is None:
self.cache = Cache(1024**2 // self.object_size)
self.cli = client.Connection(authurl, username, password)
def __init__(self, auth, container, object_size, objects, cache=None, read_only=False):
self.container = container
self.object_size = object_size
self.objects = objects
self.pos = 0
self.locked = False
self.meta = dict()
self.read_only = read_only
self.bytes_in = 0
self.bytes_out = 0
self.cache = cache
if self.cache is None:
self.cache = Cache(1024**2 // self.object_size)
self.cli = client.Connection(**auth)
class SwiftStorage(object):
"""
Manages a object-split file stored in OpenStack Object Storage (swift).
May raise StorageError (IOError).
"""
def __init__(self, auth, container, object_size, objects, cache=None, read_only=False):
self.container = container
self.object_size = object_size
self.objects = objects
self.pos = 0
self.locked = False
self.meta = dict()
self.read_only = read_only
self.bytes_in = 0
self.bytes_out = 0
self.cache = cache
if self.cache is None:
self.cache = Cache(1024**2 // self.object_size)
self.cli = client.Connection(**auth)
def __str__(self):
return self.container
def lock(self, client_id):
"""Set the storage as busy"""
if self.locked:
return
try:
headers, _ = self.cli.get_container(self.container)
except (socket.error, client.ClientException) as ex:
raise StorageError(errno.EIO, "Failed to lock: %s" % ex)
self.meta = getMeta(headers)
if self.meta.get('client'):
raise StorageError(errno.EBUSY, "Already in use: %s" % self.meta['client'])
self.meta['client'] = "%s@%i" % (client_id, time())
hdrs = setMeta(self.meta)
try:
self.cli.put_container(self.container, headers=hdrs)
except (socket.error, client.ClientException) as ex:
raise StorageError(errno.EIO, "Failed to lock: %s" % ex)
self.locked = True
def unlock(self):
"""Set the storage as free"""
if not self.locked:
return
self.meta['last'] = self.meta.get('client')
self.meta['client'] = ''
hdrs = setMeta(self.meta)
try:
self.cli.put_container(self.container, headers=hdrs)
except (socket.error, client.ClientException) as ex:
raise StorageError(errno.EIO, "Failed to unlock: %s" % ex)
self.locked = False
def read(self, size):
data = bytearray()
_size = size
while _size > 0:
obj = self.fetch_object(self.object_num)
if obj == b'':
break
if _size + self.object_pos >= self.object_size:
data += obj[self.object_pos:]
part_size = self.object_size - self.object_pos
else:
data += obj[self.object_pos:self.object_pos+_size]
part_size = _size
_size -= part_size
self.seek(self.pos + part_size)
return data
def write(self, data):
if self.read_only:
raise StorageError(errno.EROFS, "Read only storage")
_data = data[:]
if self.object_pos != 0:
# object-align the beginning of data
obj = self.fetch_object(self.object_num)
_data = obj[:self.object_pos] + _data
self.seek(self.pos - self.object_pos)
reminder = len(_data) % self.object_size
if reminder != 0:
# object-align the end of data
obj = self.fetch_object(self.object_num + (len(_data) // self.object_size))
_data += obj[reminder:]
assert len(_data) % self.object_size == 0, "Data not aligned!"
offs = 0
object_num = self.object_num
while offs < len(_data):
self.put_object(object_num, _data[offs:offs+self.object_size])
offs += self.object_size
object_num += 1
def tell(self):
return self.pos
@property
def object_pos(self):
# position in the object
return self.pos % self.object_size
@property
def object_num(self):
# object number based on the position
return self.pos // self.object_size
@property
def size(self):
return self.object_size * self.objects
def flush(self):
self.cache.flush()
def object_name(self, object_num):
return "disk.part/%08i" % object_num
def fetch_object(self, object_num):
if object_num >= self.objects:
return b''
data = self.cache.get(object_num)
if not data:
object_name = self.object_name(object_num)
try:
_, data = self.cli.get_object(self.container, object_name)
except socket.error as ex:
raise StorageError(errno.EIO, ex)
except client.ClientException as ex:
if ex.http_status != 404:
raise StorageError(errno.EIO, ex)
return b'\0' * self.object_size
if len(data) != self.object_size:
raise StorageError(errno.EIO,
"Invalid object size (%s), %s expected" % len(data), self.object_size
)
self.bytes_in += self.object_size
self.cache.set(object_num, data)
return data
def put_object(self, object_num, data):
if object_num >= self.objects:
raise StorageError(errno.ESPIPE, "Write offset out of bounds")
object_name = self.object_name(object_num)
try:
etag = self.cli.put_object(self.container, object_name, data)
except (socket.error, client.ClientException) as ex:
raise StorageError(errno.EIO, ex)
checksum = md5(data).hexdigest()
etag = etag.lower()
if etag != checksum:
raise StorageError(errno.EAGAIN, "Block integrity error (object_num=%s)" % object_num)
self.bytes_out += self.object_size
self.cache.set(object_num, data)
def seek(self, offset):
if offset < 0 or offset > self.size:
raise StorageError(errno.ESPIPE, "Offset out of bounds")
self.pos = offset
class SwiftStorage(object):
"""
Manages a object-split file stored in OpenStack Object Storage (swift).
May raise StorageError (IOError).
"""
def __init__(self,
auth,
container,
object_size,
objects,
cache=None,
read_only=False):
self.container = container
self.object_size = object_size
self.objects = objects
self.pos = 0
self.locked = False
self.meta = dict()
self.read_only = read_only
self.bytes_in = 0
self.bytes_out = 0
self.cache = cache
if self.cache is None:
self.cache = Cache(1024**2 // self.object_size)
self.cli = client.Connection(**auth)
def __str__(self):
return self.container
def lock(self, client_id):
"""Set the storage as busy"""
if self.locked:
return
try:
headers, _ = self.cli.get_container(self.container)
except (socket.error, client.ClientException) as ex:
raise StorageError(errno.EIO, "Failed to lock: %s" % ex)
self.meta = getMeta(headers)
if self.meta.get('client'):
raise StorageError(errno.EBUSY,
"Already in use: %s" % self.meta['client'])
self.meta['client'] = "%s@%i" % (client_id, time())
hdrs = setMeta(self.meta)
try:
self.cli.put_container(self.container, headers=hdrs)
except (socket.error, client.ClientException) as ex:
raise StorageError(errno.EIO, "Failed to lock: %s" % ex)
self.locked = True
def unlock(self):
"""Set the storage as free"""
if not self.locked:
return
self.meta['last'] = self.meta.get('client')
self.meta['client'] = ''
hdrs = setMeta(self.meta)
try:
self.cli.put_container(self.container, headers=hdrs)
except (socket.error, client.ClientException) as ex:
raise StorageError(errno.EIO, "Failed to unlock: %s" % ex)
self.locked = False
def read(self, size):
data = bytearray()
_size = size
while _size > 0:
obj = self.fetch_object(self.object_num)
if obj == b'':
break
if _size + self.object_pos >= self.object_size:
data += obj[self.object_pos:]
part_size = self.object_size - self.object_pos
else:
data += obj[self.object_pos:self.object_pos + _size]
part_size = _size
_size -= part_size
self.seek(self.pos + part_size)
return data
def write(self, data):
if self.read_only:
raise StorageError(errno.EROFS, "Read only storage")
_data = data[:]
if self.object_pos != 0:
# object-align the beginning of data
obj = self.fetch_object(self.object_num)
_data = obj[:self.object_pos] + _data
self.seek(self.pos - self.object_pos)
reminder = len(_data) % self.object_size
if reminder != 0:
# object-align the end of data
obj = self.fetch_object(self.object_num +
(len(_data) // self.object_size))
_data += obj[reminder:]
assert len(_data) % self.object_size == 0, "Data not aligned!"
offs = 0
object_num = self.object_num
while offs < len(_data):
self.put_object(object_num, _data[offs:offs + self.object_size])
offs += self.object_size
object_num += 1
def tell(self):
return self.pos
@property
def object_pos(self):
# position in the object
return self.pos % self.object_size
@property
def object_num(self):
# object number based on the position
return self.pos // self.object_size
@property
def size(self):
return self.object_size * self.objects
def flush(self):
self.cache.flush()
def object_name(self, object_num):
return "disk.part/%08i" % object_num
def fetch_object(self, object_num):
if object_num >= self.objects:
return b''
data = self.cache.get(object_num)
if not data:
object_name = self.object_name(object_num)
try:
_, data = self.cli.get_object(self.container, object_name)
except socket.error as ex:
raise StorageError(errno.EIO, ex)
except client.ClientException as ex:
if ex.http_status != 404:
raise StorageError(errno.EIO, ex)
return b'\0' * self.object_size
if len(data) != self.object_size:
raise StorageError(
errno.EIO,
"Invalid object size (%s), %s expected" % len(data),
self.object_size)
self.bytes_in += self.object_size
self.cache.set(object_num, data)
return data
def put_object(self, object_num, data):
if object_num >= self.objects:
raise StorageError(errno.ESPIPE, "Write offset out of bounds")
object_name = self.object_name(object_num)
try:
etag = self.cli.put_object(self.container, object_name, data)
except (socket.error, client.ClientException) as ex:
raise StorageError(errno.EIO, ex)
checksum = md5(data).hexdigest()
etag = etag.lower()
if etag != checksum:
raise StorageError(
errno.EAGAIN,
"Block integrity error (object_num=%s)" % object_num)
self.bytes_out += self.object_size
self.cache.set(object_num, data)
def seek(self, offset):
if offset < 0 or offset > self.size:
raise StorageError(errno.ESPIPE, "Offset out of bounds")
self.pos = offset
def setUp(self):
from swiftnbd.cache import Cache
self.cache = Cache(10)
class CacheTestCase(unittest.TestCase):
"""Test the cache class."""
def setUp(self):
from swiftnbd.cache import Cache
self.cache = Cache(10)
def tearDown(self):
pass
def test_get_miss(self):
data = self.cache.get(1)
self.assertEqual(self.cache.ref[1], 0)
self.assertEqual(data, None)
def test_get_hit(self):
self.cache.set(1, "DATA1")
self.cache.set(2, "DATA2")
data = self.cache.get(1)
self.assertEqual(data, "DATA1")
# set + get = 2 refferences
self.assertEqual(self.cache.ref[1], 2)
data = self.cache.get(2)
self.assertEqual(data, "DATA2")
def test_set(self):
self.cache.set(1, "1")
self.assertEqual(self.cache.ref[1], 1)
self.cache.set(1, "1")
self.assertEqual(self.cache.ref[1], 2)
def test_limit(self):
for i in range(10):
self.cache.set(i, "DATA%s" % i)
self.assertEqual(len(self.cache.data), 10)
for i in range(10):
self.assertEqual(self.cache.ref[i], 1)
for i in range(10):
for j in range(i+1):
self.cache.get(i)
# 0 should be discarded
self.cache.set(10, "DATA11")
self.assertEqual(len(self.cache.data), 10)
self.assertEqual(self.cache.ref[0], 0)
self.assertTrue(0 not in self.cache.data)
# 10 should be discarded
self.cache.set(11, "DATA12")
self.assertEqual(len(self.cache.data), 10)
self.assertEqual(self.cache.ref[10], 0)
self.assertTrue(10 not in self.cache.data)
def run(self):
if os.path.isfile(self.args.pidfile):
self.log.error("%s found: is the server already running?" %
self.args.pidfile)
return 1
stores = dict()
for container, values in self.conf.items():
auth = dict(
authurl=self.args.authurl,
user=values['username'],
key=values['password'],
)
if self.args.keystone:
try:
from keystoneclient.v2_0 import client as _check_for_ksclient
except ImportError:
sys.exit(
"auth 2.0 (keystone) requires python-keystoneclient")
else:
self.log.debug("using auth 2.0 (keystone)")
if self.args.keystone_separator not in values['username']:
self.log.error("%s: separator not found in %r, skipping" %
(container, values['username']))
continue
keystone_auth = values['username'].split(
self.args.keystone_separator, 1)
auth['tenant_name'], auth['user'] = keystone_auth
auth['auth_version'] = '2.0'
auth['os_options'] = dict(
service_type=self.args.keystone_service,
endpoint_type=self.args.keystone_endpoint,
region_name=self.args.keystone_region,
)
self.log.debug("os_options: %r" % auth['os_options'])
cli = client.Connection(**auth)
try:
headers, _ = cli.get_container(container)
except (socket.error, client.ClientException) as ex:
if getattr(ex, 'http_status', None) == 404:
self.log.warning("%s doesn't exist, skipping" % container)
continue
else:
self.log.error("%s: %r, skipping" % (container, ex.msg))
continue
self.log.debug(headers)
meta = getMeta(headers)
if not meta:
self.log.warning("%s doesn't appear to be setup, skipping" %
container)
continue
self.log.debug("Meta: %s" % meta)
try:
object_size = int(meta['object-size'])
objects = int(meta['objects'])
except ValueError as ex:
self.log.error("%s doesn't appear to be correct: %s" %
(container, ex))
return 1
if meta['version'] != disk_version:
self.log.warning("Version mismatch %s != %s in %s" %
(meta['version'], disk_version, container))
stores[container] = SwiftStorage(
auth,
container,
object_size,
objects,
Cache(int(self.args.cache_limit * 1024**2 / object_size)),
values['read-only'].lower() in ('1', 'yes', 'true', 'on'),
)
addr = (self.args.bind_address, self.args.bind_port)
server = Server(addr, stores)
if not self.args.foreground:
try:
if os.fork() != 0:
os._exit(0)
except OSError as ex:
self.log.error("Failed to daemonize: %s" % ex)
return 1
os.setsid()
fd = os.open(os.devnull, os.O_RDWR)
os.dup2(fd, sys.stdin.fileno())
os.dup2(fd, sys.stdout.fileno())
os.dup2(fd, sys.stderr.fileno())
self.log.info("Starting to serve on %s:%s" % (addr[0], addr[1]))
try:
fd = os.open(self.args.pidfile,
(os.O_CREAT | os.O_EXCL | os.O_WRONLY), 0o644)
except OSError as ex:
self.log.error("Failed to create the pidfile: %s" % ex)
return 1
with os.fdopen(fd, "w") as pidfile_handle:
pidfile_handle.write("%s\n" % os.getpid())
pidfile_handle.flush()
server.serve_forever()
os.remove(self.args.pidfile)
# unlock the storages before exit
server.unlock_all()
self.log.info("Exiting...")
return 0
def setUp(self):
from swiftnbd.cache import Cache
self.cache = Cache(10)
class CacheTestCase(unittest.TestCase):
"""Test the cache class."""
def setUp(self):
from swiftnbd.cache import Cache
self.cache = Cache(10)
def tearDown(self):
pass
def test_get_miss(self):
data = self.cache.get(1)
self.assertEqual(self.cache.ref[1], 0)
self.assertEqual(data, None)
def test_get_hit(self):
self.cache.set(1, "DATA1")
self.cache.set(2, "DATA2")
data = self.cache.get(1)
self.assertEqual(data, "DATA1")
# set + get = 2 refferences
self.assertEqual(self.cache.ref[1], 2)
data = self.cache.get(2)
self.assertEqual(data, "DATA2")
def test_set(self):
self.cache.set(1, "1")
self.assertEqual(self.cache.ref[1], 1)
self.cache.set(1, "1")
self.assertEqual(self.cache.ref[1], 2)
def test_limit(self):
for i in range(10):
self.cache.set(i, "DATA%s" % i)
self.assertEqual(len(self.cache.data), 10)
for i in range(10):
self.assertEqual(self.cache.ref[i], 1)
for i in range(10):
for j in range(i + 1):
self.cache.get(i)
# 0 should be discarded
self.cache.set(10, "DATA11")
self.assertEqual(len(self.cache.data), 10)
self.assertEqual(self.cache.ref[0], 0)
self.assertTrue(0 not in self.cache.data)
# 10 should be discarded
self.cache.set(11, "DATA12")
self.assertEqual(len(self.cache.data), 10)
self.assertEqual(self.cache.ref[10], 0)
self.assertTrue(10 not in self.cache.data)
def run(self):
if os.path.isfile(self.args.pidfile):
self.log.error("%s found: is the server already running?" %
self.args.pidfile)
return 1
stores = dict()
for container, values in self.conf.items():
cli = client.Connection(values['authurl'], values['username'],
values['password'])
try:
headers, _ = cli.get_container(container)
except (socket.error, client.ClientException) as ex:
if getattr(ex, 'http_status', None) == 404:
self.log.warning("%s doesn't exist, skipping" % container)
continue
else:
self.log.error(ex)
return 1
self.log.debug(headers)
meta = getMeta(headers)
if not meta:
self.log.warning("%s doesn't appear to be setup, skipping" %
container)
continue
self.log.debug("Meta: %s" % meta)
try:
object_size = int(meta['object-size'])
objects = int(meta['objects'])
except ValueError as ex:
self.log.error("%s doesn't appear to be correct: %s" %
(container, ex))
return 1
if meta['version'] != disk_version:
self.log.warning("Version mismatch %s != %s in %s" %
(meta['version'], disk_version, container))
stores[container] = SwiftStorage(
values['authurl'],
values['username'],
values['password'],
container,
object_size,
objects,
Cache(int(self.args.cache_limit * 1024**2 / object_size)),
values['read-only'].lower() in ('1', 'yes', 'true', 'on'),
)
addr = (self.args.bind_address, self.args.bind_port)
server = Server(addr, stores)
if not self.args.foreground:
try:
if os.fork() != 0:
os._exit(0)
except OSError as ex:
self.log.error("Failed to daemonize: %s" % ex)
return 1
os.setsid()
fd = os.open(os.devnull, os.O_RDWR)
os.dup2(fd, sys.stdin.fileno())
os.dup2(fd, sys.stdout.fileno())
os.dup2(fd, sys.stderr.fileno())
self.log.info("Starting to serve on %s:%s" % (addr[0], addr[1]))
try:
fd = os.open(self.args.pidfile,
(os.O_CREAT | os.O_EXCL | os.O_WRONLY), 0o644)
except OSError as ex:
self.log.error("Failed to create the pidfile: %s" % ex)
return 1
with os.fdopen(fd, "w") as pidfile_handle:
pidfile_handle.write("%s\n" % os.getpid())
pidfile_handle.flush()
server.serve_forever()
os.remove(self.args.pidfile)
# unlock the storages before exit
server.unlock_all()
self.log.info("Exiting...")
return 0
