def test_import(self):
benji_obj = self.benjiOpen(initdb=True)
benji_obj.import_(StringIO(self.IMPORT))
version = benji_obj.ls(version_uid=VersionUid(1))[0]
self.assertTrue(isinstance(version.uid, VersionUid))
self.assertEqual(1, version.uid)
self.assertEqual('data-backup', version.name)
self.assertEqual('snapshot-name', version.snapshot_name)
self.assertEqual(4096, version.block_size)
self.assertTrue(version.valid)
self.assertFalse(version.protected)
self.assertIsInstance(version.blocks, list)
self.assertIsInstance(version.tags, list)
self.assertEqual({'b_daily', 'b_weekly', 'b_monthly'}, set([tag.name for tag in version.tags]))
self.assertEqual(datetime.datetime.strptime('2018-05-16T11:57:10', '%Y-%m-%dT%H:%M:%S'), version.date)
blocks = benji_obj.ls_version(VersionUid(1))
self.assertTrue(len(blocks) > 0)
max_i = len(blocks) - 1
for i, block in enumerate(blocks):
self.assertEqual(VersionUid(1), block.version_uid)
self.assertEqual(i, block.id)
if i != max_i:
self.assertEqual(4096, block.size)
self.assertEqual(datetime.datetime.strptime('2018-05-16T11:57:10', '%Y-%m-%dT%H:%M:%S'), block.date)
self.assertTrue(block.valid)
benji_obj.close()
def test_lock_version_context_manager(self):
locking = self.metadata_backend.locking()
with locking.with_version_lock(VersionUid(1), reason='locking test'):
with self.assertRaises(InternalError):
locking.lock_version(VersionUid(1), reason='locking test')
locking.lock_version(VersionUid(1), reason='locking test')
locking.unlock_version(VersionUid(1))
def test_lock_version(self):
locking = self.metadata_backend.locking()
locking.lock_version(VersionUid(1), reason='locking test')
self.assertRaises(
InternalError,
lambda: locking.lock_version(VersionUid(1), reason='locking test'))
locking.unlock_version(VersionUid(1))
def test_version_uid_create_from_readable(self):
self.assertEqual(VersionUid(1), VersionUid.create_from_readables(1))
self.assertEqual(VersionUid(1), VersionUid.create_from_readables('V1'))
uids = VersionUid.create_from_readables(['V1'])
self.assertTrue(isinstance(uids, list))
self.assertTrue(len(uids) == 1)
self.assertEqual(VersionUid(1), uids[0])
uids = VersionUid.create_from_readables(['V1', 'V2', 3])
self.assertTrue(isinstance(uids, list))
self.assertTrue(len(uids) == 3)
self.assertEqual(VersionUid(1), uids[0])
self.assertEqual(VersionUid(2), uids[1])
self.assertEqual(VersionUid(3), uids[2])
def backup(self,
name,
snapshot_name,
source,
rbd,
base_version_uid,
block_size=None,
tags=None):
base_version_uid = VersionUid.create_from_readables(base_version_uid)
benji_obj = None
try:
benji_obj = Benji(self.config, block_size=block_size)
hints = None
if rbd:
data = ''.join(
[line for line in fileinput.input(rbd).readline()])
hints = hints_from_rbd_diff(data)
backup_version_uid = benji_obj.backup(name, snapshot_name, source,
hints, base_version_uid,
tags)
if self.machine_output:
benji_obj.export_any(
{'versions': benji_obj.ls(version_uid=backup_version_uid)},
sys.stdout,
ignore_relationships=[((Version, ), ('blocks', ))])
finally:
if benji_obj:
benji_obj.close()
def deep_scrub(self, version_uid, source, block_percentage):
version_uid = VersionUid.create_from_readables(version_uid)
if block_percentage:
block_percentage = int(block_percentage)
benji_obj = None
try:
benji_obj = Benji(self.config)
benji_obj.deep_scrub(version_uid,
source=source,
block_percentage=block_percentage)
except benji.exception.ScrubbingError:
if self.machine_output:
benji_obj.export_any(
{
'versions': benji_obj.ls(version_uid=version_uid),
'errors': benji_obj.ls(version_uid=version_uid)
},
sys.stdout,
ignore_relationships=[((Version, ), ('blocks', ))])
raise
else:
if self.machine_output:
benji_obj.export_any(
{
'versions': benji_obj.ls(version_uid=version_uid),
'errors': []
},
sys.stdout,
ignore_relationships=[((Version, ), ('blocks', ))])
finally:
if benji_obj:
benji_obj.close()
def stats(self, version_uid, limit=None):
version_uid = VersionUid.create_from_readables(version_uid)
if limit:
limit = int(limit)
if limit <= 0:
raise benji.exception.UsageError(
'Limit has to be a positive integer.')
benji_obj = None
try:
benji_obj = Benji(self.config)
stats = benji_obj.stats(version_uid, limit)
if self.machine_output:
stats = list(
stats) # resolve iterator, otherwise it's not serializable
benji_obj.export_any(
{'stats': stats},
sys.stdout,
)
else:
self._stats_tbl_output(stats)
finally:
if benji_obj:
benji_obj.close()
def diff_meta(self, version_uid1, version_uid2):
""" Output difference between two version in blocks.
"""
version_uid1 = VersionUid.create_from_readables(version_uid1)
version_uid2 = VersionUid.create_from_readables(version_uid2)
# TODO: Feel free to create a default diff format.
benji_obj = None
try:
benji_obj = Benji(self.config)
# Check if versions exist
blocks1 = benji_obj.ls_version(version_uid1)
if not blocks1:
raise FileNotFoundError('Version {} doesn\'t exist.'.format(
version_uid1.readable))
blocks2 = benji_obj.ls_version(version_uid2)
if not blocks2:
raise FileNotFoundError('Version {} doesn\'t exist.'.format(
version_uid2.readable))
max_len = max(len(blocks1), len(blocks2))
for i in range(max_len):
b1 = b2 = None
try:
b1 = blocks1.pop(0)
except IndexError:
pass
try:
b2 = blocks2.pop(0)
except IndexError:
pass
if b1 and b2:
assert b1.id == b2.id
try:
if b1.uid == b2.uid:
print('SAME {}'.format(b1.id))
elif b1 is None and b2:
print('NEW RIGHT {}'.format(b2.id))
elif b1 and b2 is None:
print('NEW LEFT {}'.format(b1.id))
else:
print('DIFF {}'.format(b1.id))
except BrokenPipeError:
pass
finally:
if benji_obj:
benji_obj.close()
def import_from_backend(self, version_uids):
version_uids = VersionUid.create_from_readables(version_uids)
benji_obj = None
try:
benji_obj = Benji(self.config)
benji_obj.import_from_backend(version_uids)
finally:
if benji_obj:
benji_obj.close()
def export_to_backend(self, version_uids, force=False):
version_uids = VersionUid.create_from_readables(version_uids)
benji_obj = None
try:
benji_obj = Benji(self.config)
benji_obj.export_to_backend(version_uids, overwrite=force)
finally:
if benji_obj:
benji_obj.close()
def test_version(self):
version_uid = VersionUid(1)
self.data_backend.save_version(version_uid, 'Hallo')
data = self.data_backend.read_version(version_uid)
self.assertEqual('Hallo', data)
version_uids = self.data_backend.list_versions()
self.assertTrue(len(version_uids) == 1)
self.assertEqual(version_uid, version_uids[0])
self.data_backend.rm_version(version_uid)
version_uids = self.data_backend.list_versions()
self.assertTrue(len(version_uids) == 0)
def protect(self, version_uids):
version_uids = VersionUid.create_from_readables(version_uids)
benji_obj = None
try:
benji_obj = Benji(self.config)
for version_uid in version_uids:
try:
benji_obj.protect(version_uid)
except benji.exception.NoChange:
logger.warning('Version {} already was protected.'.format(
version_uid))
finally:
if benji_obj:
benji_obj.close()
def rm_tag(self, version_uid, names):
version_uid = VersionUid.create_from_readables(version_uid)
benji_obj = None
try:
benji_obj = Benji(self.config)
for name in names:
try:
benji_obj.rm_tag(version_uid, name)
except benji.exception.NoChange:
logger.warning('Version {} has no tag {}.'.format(
version_uid, name))
finally:
if benji_obj:
benji_obj.close()
def restore(self,
version_uid,
target,
sparse,
force,
metadata_backend_less=False):
version_uid = VersionUid.create_from_readables(version_uid)
benji_obj = None
try:
benji_obj = Benji(self.config, in_memory=metadata_backend_less)
if metadata_backend_less:
benji_obj.import_from_backend([version_uid])
benji_obj.restore(version_uid, target, sparse, force)
finally:
if benji_obj:
benji_obj.close()
def export(self, version_uids, output_file=None, force=False):
version_uids = VersionUid.create_from_readables(version_uids)
benji_obj = None
try:
benji_obj = Benji(self.config)
if output_file is None:
benji_obj.export(version_uids, sys.stdout)
else:
if os.path.exists(output_file) and not force:
raise FileExistsError('The output file already exists.')
with open(output_file, 'w') as f:
benji_obj.export(version_uids, f)
finally:
if benji_obj:
benji_obj.close()
def rm(self, version_uids, force, keep_backend_metadata):
version_uids = VersionUid.create_from_readables(version_uids)
disallow_rm_when_younger_than_days = self.config.get(
'disallowRemoveWhenYounger', types=int)
benji_obj = None
try:
benji_obj = Benji(self.config)
for version_uid in version_uids:
benji_obj.rm(version_uid,
force=force,
disallow_rm_when_younger_than_days=
disallow_rm_when_younger_than_days,
keep_backend_metadata=keep_backend_metadata)
finally:
if benji_obj:
benji_obj.close()
def test(self):
benji_obj = self.benjiOpen(initdb=False)
store = BenjiStore(benji_obj)
addr = ('127.0.0.1', self.SERVER_PORT)
read_only = False
self.nbd_server = NbdServer(addr, store, read_only)
logger.info("Starting to serve nbd on %s:%s" % (addr[0], addr[1]))
self.subprocess_run(args=['sudo', 'modprobe', 'nbd'])
self.nbd_client_thread = threading.Thread(target=self.nbd_client,
daemon=True,
args=(self.version_uid, ))
self.nbd_client_thread.start()
self.nbd_server.serve_forever()
self.nbd_client_thread.join()
self.assertEqual(
{self.version_uid[0], VersionUid(2)},
set([version.uid for version in benji_obj.ls()]))
benji_obj.close()
def test_version_uid_to_key(self):
for i in range(100):
version_uid = VersionUid(random.randint(1, pow(2, 32) - 1))
key = self.data_backend._version_uid_to_key(version_uid)
version_uid_2 = self.data_backend._key_to_version_uid(key)
self.assertEqual(version_uid, version_uid_2)
def test_is_version_locked(self):
locking = self.metadata_backend.locking()
lock = locking.lock_version(VersionUid(1), reason='locking test')
self.assertTrue(locking.is_version_locked(VersionUid(1)))
locking.unlock_version(VersionUid(1))
self.assertFalse(locking.is_version_locked(VersionUid(1)))
def test_metabackend_version_not_found(self):
backend = self.metadata_backend
self.assertRaises(KeyError,
lambda: backend.get_version(VersionUid(123)))
def _key_to_version_uid(self, key):
vpl = len(self._VERSIONS_PREFIX)
vl = len(VersionUid(1).readable)
if len(key) != vpl + vl + 4:
raise RuntimeError('Invalid key name {}'.format(key))
return VersionUid.create_from_readables(key[vpl + 4:vpl + vl + 4])
def handler(self, reader, writer):
"""Handle the connection"""
try:
host, port = writer.get_extra_info("peername")
version, cow_version = None, None
self.log.info("Incoming connection from %s:%s" % (host, port))
# initial handshake
writer.write(b"NBDMAGIC" + struct.pack(">QH", self.NBD_HANDSHAKE, self.NBD_HANDSHAKE_FLAGS))
yield from writer.drain()
data = yield from reader.readexactly(4)
try:
client_flag = struct.unpack(">L", data)[0]
except struct.error:
raise IOError("Handshake failed, disconnecting")
# we support both fixed and unfixed new-style handshake
if client_flag == 0:
fixed = False
self.log.warning("Client using new-style non-fixed handshake")
elif client_flag & 1:
fixed = True
else:
raise IOError("Handshake failed, disconnecting")
# negotiation phase
while True:
header = yield from reader.readexactly(16)
try:
(magic, opt, length) = struct.unpack(">QLL", header)
except struct.error:
raise IOError("Negotiation failed: Invalid request, disconnecting")
if magic != self.NBD_HANDSHAKE:
raise IOError("Negotiation failed: bad magic number: %s" % magic)
if length:
data = yield from reader.readexactly(length)
if (len(data) != length):
raise IOError("Negotiation failed: %s bytes expected" % length)
else:
data = None
self.log.debug("[%s:%s]: opt=%s, len=%s, data=%s" % (host, port, opt, length, data))
if opt == self.NBD_OPT_EXPORTNAME:
if not data:
raise IOError("Negotiation failed: no export name was provided")
data = data.decode("ascii")
data = VersionUid.create_from_readables(data)
if data not in [v.uid for v in self.store.get_versions()]:
if not fixed:
raise IOError("Negotiation failed: unknown export name")
writer.write(struct.pack(">QLLL", self.NBD_REPLY, opt, self.NBD_REP_ERR_UNSUP, 0))
yield from writer.drain()
continue
# we have negotiated a version and it will be used
# until the client disconnects
version = self.store.get_versions(version_uid=data)[0]
self.store.open(version, self.read_only)
self.log.info("[%s:%s] Negotiated export: %s" % (host, port, version.uid))
export_flags = self.NBD_EXPORT_FLAGS
if self.read_only:
export_flags ^= self.NBD_FLAG_READ_ONLY
self.log.info("nbd is read only.")
else:
self.log.info("nbd is read/write.")
# In case size is not a multiple of 4096 we extend it to the the maximum support block
# size of 4096
size = math.ceil(version.size / 4096) * 4096
writer.write(struct.pack('>QH', size, export_flags))
writer.write(b"\x00" * 124)
yield from writer.drain()
# Transition to transmission phase
break
elif opt == self.NBD_OPT_LIST:
for version in self.store.get_versions():
version_encoded = version.uid.readable.encode("ascii")
writer.write(
struct.pack(">QLLL", self.NBD_REPLY, opt, self.NBD_REP_SERVER,
len(version_encoded) + 4))
writer.write(struct.pack(">L", len(version_encoded)))
writer.write(version_encoded)
yield from writer.drain()
writer.write(struct.pack(">QLLL", self.NBD_REPLY, opt, self.NBD_REP_ACK, 0))
yield from writer.drain()
elif opt == self.NBD_OPT_ABORT:
writer.write(struct.pack(">QLLL", self.NBD_REPLY, opt, self.NBD_REP_ACK, 0))
yield from writer.drain()
raise NbdServerAbortedNegotiationError()
else:
# we don't support any other option
if not fixed:
raise IOError("Unsupported option")
writer.write(struct.pack(">QLLL", self.NBD_REPLY, opt, self.NBD_REP_ERR_UNSUP, 0))
yield from writer.drain()
# operation phase
while True:
header = yield from reader.readexactly(28)
try:
(magic, cmd, handle, offset, length) = struct.unpack(">LLQQL", header)
except struct.error:
raise IOError("Invalid request, disconnecting")
if magic != self.NBD_REQUEST:
raise IOError("Bad magic number, disconnecting")
self.log.debug(
"[%s:%s]: cmd=%s, handle=%s, offset=%s, len=%s" % (host, port, cmd, handle, offset, length))
if cmd == self.NBD_CMD_DISC:
self.log.info("[%s:%s] disconnecting" % (host, port))
break
elif cmd == self.NBD_CMD_WRITE:
data = yield from reader.readexactly(length)
if (len(data) != length):
raise IOError("%s bytes expected, disconnecting" % length)
if self.read_only:
yield from self.nbd_response(writer, handle, error=errno.EPERM)
continue
if not cow_version:
cow_version = self.store.get_cow_version(version)
try:
self.store.write(cow_version, offset, data)
except Exception as exception:
self.log.error("[%s:%s] NBD_CMD_WRITE: %s\n%s" % (host, port, exception, traceback.format_exc()))
yield from self.nbd_response(
writer, handle, error=exception.errno if hasattr(exception, 'errno') else errno.EIO)
continue
yield from self.nbd_response(writer, handle)
elif cmd == self.NBD_CMD_READ:
try:
data = self.store.read(version, cow_version, offset, length)
except Exception as exception:
self.log.error("[%s:%s] NBD_CMD_READ: %s\n%s" % (host, port, exception, traceback.format_exc()))
yield from self.nbd_response(
writer, handle, error=exception.errno if hasattr(exception, 'errno') else errno.EIO)
continue
yield from self.nbd_response(writer, handle, data=data)
elif cmd == self.NBD_CMD_FLUSH:
# Return success right away when we're read only or when we haven't written anythin yet.
if self.read_only or not cow_version:
yield from self.nbd_response(writer, handle)
continue
try:
self.store.flush(cow_version)
except Exception as exception:
self.log.error("[%s:%s] NBD_CMD_FLUSH: %s\n%s" % (host, port, exception, traceback.format_exc()))
yield from self.nbd_response(
writer, handle, error=exception.errno if hasattr(exception, 'errno') else errno.EIO)
continue
yield from self.nbd_response(writer, handle)
else:
self.log.warning("[%s:%s] Unknown cmd %s, disconnecting" % (host, port, cmd))
break
except NbdServerAbortedNegotiationError:
self.log.info("[%s:%s] Client aborted negotiation" % (host, port))
except (asyncio.IncompleteReadError, IOError) as exception:
self.log.error("[%s:%s] %s" % (host, port, exception))
finally:
if cow_version:
self.store.fixate(cow_version)
self.store.close(version)
writer.close()