def setUp(self):
# monkey-patch swiftclient to use out mock up
import swiftnbd.swift as swift
swift.client = MockConnection
from swiftnbd.swift import SwiftStorage
# create a disk doubling the actual size of the mock up so we
# have some uninitialised space to run tests
self.store = SwiftStorage('url', 'user', 'pass', 'container', 512, 16)
self.store.flush()
def setUp(self):
# monkey-patch swiftclient to use out mock up
import swiftnbd.swift as swift
swift.client = MockConnection
from swiftnbd.swift import SwiftStorage
# create a disk doubling the actual size of the mock up so we
# have some uninitialised space to run tests
self.store = SwiftStorage('url', 'user', 'pass', 'container', 512, 16)
self.store.flush()
def do_download(self):
self.log.debug("downloading %s" % self.args.container)
cli, meta = self._setup_client()
if cli is None:
return 1
elif 'client' in meta:
self.log.error("%s is locked, downloading a container in use is unreliable" % self.args.container)
return 1
object_size = int(meta['object-size'])
objects = int(meta['objects'])
store = SwiftStorage(self.authurl,
self.username,
self.password,
self.args.container,
object_size,
objects,
)
try:
store.lock("ctl-download")
except StorageError as ex:
self.log.error(ex)
return 1
size = 0
fdo = None
try:
fdo = open(self.args.image, "w")
while True:
data = store.read(object_size)
if data == '':
break
fdo.write(data)
size += len(data)
if not self.args.quiet:
sys.stdout.write("\rDownloading %s [%.2d%%]" % (self.args.container, 100*size/(objects*object_size)))
sys.stdout.flush()
except IOError as ex:
self.log.error(ex)
return 1
except KeyboardInterrupt:
self.log.warning("user interrupt")
return 1
finally:
if fdo:
fdo.close()
try:
store.unlock()
except StorageError as ex:
self.log.warning("Failed to unlock %s: %s" % (self.args.container, ex))
if not self.args.quiet:
sys.stdout.write("\r")
sys.stdout.flush()
self.log.info("Done, %s bytes written" % size)
return 0
class SwiftStorageTestCase(unittest.TestCase):
"""Test the object-split file class."""
def setUp(self):
# monkey-patch swiftclient to use out mock up
import swiftnbd.swift as swift
swift.client = MockConnection
from swiftnbd.swift import SwiftStorage
# create a disk doubling the actual size of the mock up so we
# have some uninitialised space to run tests
self.store = SwiftStorage('url', 'user', 'pass', 'container', 512, 16)
self.store.flush()
def tearDown(self):
pass
def test_read_full_object_content(self):
self.store.seek(0)
data = self.store.read(512)
self.assertEqual(data, b'\xff' * 512)
def test_read_full_object_no_content(self):
self.store.seek(8 * 512)
data = self.store.read(512)
self.assertEqual(data, b'\0' * 512)
def test_write_full_object(self):
self.store.seek(0)
self.store.write(b'X' * 512)
self.assertEqual(MockConnection.object(0), b'X' * 512)
self.store.seek(8 * 512)
self.store.write(b'X' * 512)
self.assertEqual(MockConnection.object(8), b'X' * 512)
def test_read_partial_object_content(self):
self.store.seek(0)
data = self.store.read(256)
self.assertEqual(data, b'\xff' * 256)
def test_read_partial_object_no_content(self):
self.store.seek(8 * 512)
data = self.store.read(256)
self.assertEqual(data, b'\0' * 256)
def test_write_partial_object_content(self):
self.store.seek(0)
self.store.write(b'X' * 256)
self.assertEqual(MockConnection.object(0), b'X' * 256 + b'\xff' * 256)
def test_write_partial_object_no_content(self):
self.store.seek(8 * 512)
self.store.write(b'X' * 256)
self.assertEqual(MockConnection.object(8), b'X' * 256 + b'\0' * 256)
def test_read_inter_object_content(self):
self.store.seek(256)
data = self.store.read(512)
self.assertEqual(data, b'\xff' * 512)
def test_read_inter_object_no_content(self):
self.store.seek(8 * 512 + 256)
data = self.store.read(512)
self.assertEqual(data, b'\0' * 512)
def test_read_inter_object_content_and_no_content(self):
self.store.seek(8 * 512 - 256)
data = self.store.read(512)
self.assertEqual(data, b'\xff' * 256 + b'\0' * 256)
def test_write_inter_object_content(self):
self.store.seek(256)
self.store.write(b'X' * 512)
self.assertEqual(MockConnection.object(0), b'\xff' * 256 + b'X' * 256)
self.assertEqual(MockConnection.object(1), b'X' * 256 + b'\xff' * 256)
def test_write_inter_object_no_content(self):
self.store.seek(8 * 512 + 256)
self.store.write(b'X' * 512)
self.assertEqual(MockConnection.object(8), b'\0' * 256 + b'X' * 256)
self.assertEqual(MockConnection.object(9), b'X' * 256 + b'\0' * 256)
def test_write_inter_object_content_and_no_content(self):
self.store.seek(8 * 512 - 256)
self.store.write(b'X' * 512)
self.assertEqual(MockConnection.object(7), b'\xff' * 256 + b'X' * 256)
self.assertEqual(MockConnection.object(8), b'X' * 256 + b'\0' * 256)
def test_seek_bad_offset(self):
self.assertRaises(IOError, self.store.seek, -1)
self.assertRaises(IOError, self.store.seek, 10000000000000)
try:
self.store.seek(-1)
except IOError as ex:
self.assertEqual(ex.errno, errno.ESPIPE)
else:
self.fail("didn't raise IOError")
def test_tell(self):
self.store.seek(0)
self.assertEqual(self.store.tell(), 0)
self.store.seek(1024)
self.assertEqual(self.store.tell(), 1024)
def test_read_end_of_disk(self):
self.store.seek(15 * 512)
data = self.store.read(1024)
self.assertEqual(len(data), 512)
def test_wite_end_of_disk(self):
self.store.seek(15 * 512)
self.assertRaises(IOError, self.store.write, b'X' * 1024)
class SwiftStorageTestCase(unittest.TestCase):
"""Test the object-split file class."""
def setUp(self):
# monkey-patch swiftclient to use out mock up
import swiftnbd.swift as swift
swift.client = MockConnection
from swiftnbd.swift import SwiftStorage
# create a disk doubling the actual size of the mock up so we
# have some uninitialised space to run tests
self.store = SwiftStorage('url', 'user', 'pass', 'container', 512, 16)
self.store.flush()
def tearDown(self):
pass
def test_read_full_object_content(self):
self.store.seek(0)
data = self.store.read(512)
self.assertEqual(data, b'\xff'*512)
def test_read_full_object_no_content(self):
self.store.seek(8*512)
data = self.store.read(512)
self.assertEqual(data, b'\0'*512)
def test_write_full_object(self):
self.store.seek(0)
self.store.write(b'X'*512)
self.assertEqual(MockConnection.object(0), b'X'*512)
self.store.seek(8*512)
self.store.write(b'X'*512)
self.assertEqual(MockConnection.object(8), b'X'*512)
def test_read_partial_object_content(self):
self.store.seek(0)
data = self.store.read(256)
self.assertEqual(data, b'\xff'*256)
def test_read_partial_object_no_content(self):
self.store.seek(8*512)
data = self.store.read(256)
self.assertEqual(data, b'\0'*256)
def test_write_partial_object_content(self):
self.store.seek(0)
self.store.write(b'X'*256)
self.assertEqual(MockConnection.object(0), b'X'*256 + b'\xff'*256)
def test_write_partial_object_no_content(self):
self.store.seek(8*512)
self.store.write(b'X'*256)
self.assertEqual(MockConnection.object(8), b'X'*256 + b'\0'*256)
def test_read_inter_object_content(self):
self.store.seek(256)
data = self.store.read(512)
self.assertEqual(data, b'\xff'*512)
def test_read_inter_object_no_content(self):
self.store.seek(8*512 + 256)
data = self.store.read(512)
self.assertEqual(data, b'\0'*512)
def test_read_inter_object_content_and_no_content(self):
self.store.seek(8*512 - 256)
data = self.store.read(512)
self.assertEqual(data, b'\xff'*256 + b'\0'*256)
def test_write_inter_object_content(self):
self.store.seek(256)
self.store.write(b'X'*512)
self.assertEqual(MockConnection.object(0), b'\xff'*256 + b'X'*256)
self.assertEqual(MockConnection.object(1), b'X'*256 + b'\xff'*256)
def test_write_inter_object_no_content(self):
self.store.seek(8*512 + 256)
self.store.write(b'X'*512)
self.assertEqual(MockConnection.object(8), b'\0'*256 + b'X'*256)
self.assertEqual(MockConnection.object(9), b'X'*256 + b'\0'*256)
def test_write_inter_object_content_and_no_content(self):
self.store.seek(8*512 - 256)
self.store.write(b'X'*512)
self.assertEqual(MockConnection.object(7), b'\xff'*256 + b'X'*256)
self.assertEqual(MockConnection.object(8), b'X'*256 + b'\0'*256)
def test_seek_bad_offset(self):
self.assertRaises(IOError, self.store.seek, -1)
self.assertRaises(IOError, self.store.seek, 10000000000000)
try:
self.store.seek(-1)
except IOError as ex:
self.assertEqual(ex.errno, errno.ESPIPE)
else:
self.fail("didn't raise IOError")
def test_tell(self):
self.store.seek(0)
self.assertEqual(self.store.tell(), 0)
self.store.seek(1024)
self.assertEqual(self.store.tell(), 1024)
def test_read_end_of_disk(self):
self.store.seek(15*512)
data = self.store.read(1024)
self.assertEqual(len(data), 512)
def test_wite_end_of_disk(self):
self.store.seek(15*512)
self.assertRaises(IOError, self.store.write, b'X'*1024)
def do_download(self):
self.log.debug("downloading %s" % self.args.container)
cli, meta = self._setup_client()
if cli is None:
return 1
elif 'client' in meta:
self.log.error(
"%s is locked, downloading a container in use is unreliable" %
self.args.container)
return 1
object_size = int(meta['object-size'])
objects = int(meta['objects'])
store = SwiftStorage(
self.auth,
self.args.container,
object_size,
objects,
)
try:
store.lock("ctl-download")
except StorageError as ex:
self.log.error(ex)
return 1
size = 0
fdo = None
try:
fdo = open(self.args.image, "wb")
while True:
data = store.read(object_size)
if data == b'':
break
fdo.write(data)
size += len(data)
if not self.args.quiet:
sys.stdout.write("\rDownloading %s [%.2d%%]" %
(self.args.container, 100 * size /
(objects * object_size)))
sys.stdout.flush()
except IOError as ex:
self.log.error(ex)
return 1
except KeyboardInterrupt:
self.log.warning("user interrupt")
return 1
finally:
if fdo:
fdo.close()
try:
store.unlock()
except StorageError as ex:
self.log.warning("Failed to unlock %s: %s" %
(self.args.container, ex))
if not self.args.quiet:
sys.stdout.write("\r")
sys.stdout.flush()
self.log.info("Done, %s bytes written" % size)
return 0
def run(self):
if os.path.isfile(self.args.pidfile):
self.log.error("%s found: is the server already running?" %
self.args.pidfile)
return 1
stores = dict()
for container, values in self.conf.items():
auth = dict(
authurl=self.args.authurl,
user=values['username'],
key=values['password'],
)
if self.args.keystone:
try:
from keystoneclient.v2_0 import client as _check_for_ksclient
except ImportError:
sys.exit(
"auth 2.0 (keystone) requires python-keystoneclient")
else:
self.log.debug("using auth 2.0 (keystone)")
if self.args.keystone_separator not in values['username']:
self.log.error("%s: separator not found in %r, skipping" %
(container, values['username']))
continue
keystone_auth = values['username'].split(
self.args.keystone_separator, 1)
auth['tenant_name'], auth['user'] = keystone_auth
auth['auth_version'] = '2.0'
auth['os_options'] = dict(
service_type=self.args.keystone_service,
endpoint_type=self.args.keystone_endpoint,
region_name=self.args.keystone_region,
)
self.log.debug("os_options: %r" % auth['os_options'])
cli = client.Connection(**auth)
try:
headers, _ = cli.get_container(container)
except (socket.error, client.ClientException) as ex:
if getattr(ex, 'http_status', None) == 404:
self.log.warning("%s doesn't exist, skipping" % container)
continue
else:
self.log.error("%s: %r, skipping" % (container, ex.msg))
continue
self.log.debug(headers)
meta = getMeta(headers)
if not meta:
self.log.warning("%s doesn't appear to be setup, skipping" %
container)
continue
self.log.debug("Meta: %s" % meta)
try:
object_size = int(meta['object-size'])
objects = int(meta['objects'])
except ValueError as ex:
self.log.error("%s doesn't appear to be correct: %s" %
(container, ex))
return 1
if meta['version'] != disk_version:
self.log.warning("Version mismatch %s != %s in %s" %
(meta['version'], disk_version, container))
stores[container] = SwiftStorage(
auth,
container,
object_size,
objects,
Cache(int(self.args.cache_limit * 1024**2 / object_size)),
values['read-only'].lower() in ('1', 'yes', 'true', 'on'),
)
addr = (self.args.bind_address, self.args.bind_port)
server = Server(addr, stores)
if not self.args.foreground:
try:
if os.fork() != 0:
os._exit(0)
except OSError as ex:
self.log.error("Failed to daemonize: %s" % ex)
return 1
os.setsid()
fd = os.open(os.devnull, os.O_RDWR)
os.dup2(fd, sys.stdin.fileno())
os.dup2(fd, sys.stdout.fileno())
os.dup2(fd, sys.stderr.fileno())
self.log.info("Starting to serve on %s:%s" % (addr[0], addr[1]))
try:
fd = os.open(self.args.pidfile,
(os.O_CREAT | os.O_EXCL | os.O_WRONLY), 0o644)
except OSError as ex:
self.log.error("Failed to create the pidfile: %s" % ex)
return 1
with os.fdopen(fd, "w") as pidfile_handle:
pidfile_handle.write("%s\n" % os.getpid())
pidfile_handle.flush()
server.serve_forever()
os.remove(self.args.pidfile)
# unlock the storages before exit
server.unlock_all()
self.log.info("Exiting...")
return 0
def run(self):
if os.path.isfile(self.args.pidfile):
self.log.error("%s found: is the server already running?" %
self.args.pidfile)
return 1
stores = dict()
for container, values in self.conf.items():
cli = client.Connection(values['authurl'], values['username'],
values['password'])
try:
headers, _ = cli.get_container(container)
except (socket.error, client.ClientException) as ex:
if getattr(ex, 'http_status', None) == 404:
self.log.warning("%s doesn't exist, skipping" % container)
continue
else:
self.log.error(ex)
return 1
self.log.debug(headers)
meta = getMeta(headers)
if not meta:
self.log.warning("%s doesn't appear to be setup, skipping" %
container)
continue
self.log.debug("Meta: %s" % meta)
try:
object_size = int(meta['object-size'])
objects = int(meta['objects'])
except ValueError as ex:
self.log.error("%s doesn't appear to be correct: %s" %
(container, ex))
return 1
if meta['version'] != disk_version:
self.log.warning("Version mismatch %s != %s in %s" %
(meta['version'], disk_version, container))
stores[container] = SwiftStorage(
values['authurl'],
values['username'],
values['password'],
container,
object_size,
objects,
Cache(int(self.args.cache_limit * 1024**2 / object_size)),
values['read-only'].lower() in ('1', 'yes', 'true', 'on'),
)
addr = (self.args.bind_address, self.args.bind_port)
server = Server(addr, stores)
if not self.args.foreground:
try:
if os.fork() != 0:
os._exit(0)
except OSError as ex:
self.log.error("Failed to daemonize: %s" % ex)
return 1
os.setsid()
fd = os.open(os.devnull, os.O_RDWR)
os.dup2(fd, sys.stdin.fileno())
os.dup2(fd, sys.stdout.fileno())
os.dup2(fd, sys.stderr.fileno())
self.log.info("Starting to serve on %s:%s" % (addr[0], addr[1]))
try:
fd = os.open(self.args.pidfile,
(os.O_CREAT | os.O_EXCL | os.O_WRONLY), 0o644)
except OSError as ex:
self.log.error("Failed to create the pidfile: %s" % ex)
return 1
with os.fdopen(fd, "w") as pidfile_handle:
pidfile_handle.write("%s\n" % os.getpid())
pidfile_handle.flush()
server.serve_forever()
os.remove(self.args.pidfile)
# unlock the storages before exit
server.unlock_all()
self.log.info("Exiting...")
return 0
