def test_salt_empty_blocks(self):
vol1 = 'vol1'
vol2 = 'vol2'
manifest1 = Manifest()
manifest2 = Manifest()
conf = LunrConfig({'backup': {'client': 'memory'}})
worker1 = Worker(vol1, conf, manifest1)
worker2 = Worker(vol1, conf, manifest2)
self.assert_(worker1.manifest.salt != worker2.manifest.salt)
self.assert_(worker1.empty_block_hash != worker2.empty_block_hash)
self.assertEquals(worker1.empty_block, worker2.empty_block)
def test_audit(self):
manifest = Manifest.blank(2)
worker = Worker('foo',
LunrConfig({
'backup': {'client': 'memory'},
'storage': {'run_dir': self.scratch}
}),
manifest=manifest)
conn = worker.conn
conn.put_container('foo')
backup = manifest.create_backup('bak1')
backup[0] = worker.empty_block_hash
conn.put_object('foo', backup[0], 'zeroes')
backup[1] = 'some_block_hash'
conn.put_object('foo', backup[1], ' more stuff')
save_manifest(manifest, conn, worker.id, worker._lock_path())
# Add some non referenced blocks.
conn.put_object('foo', 'stuff1', 'unreferenced stuff1')
conn.put_object('foo', 'stuff2', 'unreferenced stuff2')
conn.put_object('foo', 'stuff3', 'unreferenced stuff3')
_headers, original_list = conn.get_container('foo')
# Manifest, 2 blocks, 3 stuffs.
self.assertEquals(len(original_list), 6)
worker.audit()
_headers, new_list = conn.get_container('foo')
# Manifest, 2 blocks.
self.assertEquals(len(new_list), 3)
def test_delete_with_missing_blocks(self):
stats_path = os.path.join(self.scratch, 'stats')
manifest = Manifest.blank(2)
worker = Worker('foo',
LunrConfig({
'backup': {'client': 'memory'},
'storage': {'run_dir': self.scratch}
}),
manifest=manifest)
conn = worker.conn
conn.put_container('foo')
backup = manifest.create_backup('bak1')
backup[0] = worker.empty_block_hash
backup[1] = 'some_random_block_that_isnt_uploaded'
save_manifest(manifest, conn, worker.id, worker._lock_path())
obj = conn.get_object('foo', 'manifest', newest=True)
self.assertRaises(ClientException, conn.get_object,
'foo', backup[0], newest=True)
self.assertRaises(ClientException, conn.get_object,
'foo', backup[1], newest=True)
# Shouldn't blow up on 404.
worker.delete('bak1')
# Manifest should still be nicely deleted.
self.assertRaises(ClientException, conn.get_object,
'foo', 'manifest', newest=True)
def save(self, snapshot, backup_id, cinder):
job_stats_path = self._stats_file(snapshot['origin'])
logger.rename('lunr.storage.helper.backup.save')
setproctitle("lunr-save: " + backup_id)
size = snapshot['size'] / 1024 / 1024 / 1024
try:
op_start = time()
worker = Worker(snapshot['origin'],
conf=self.conf,
stats_path=job_stats_path)
except exc.ClientException, e:
if e.http_status != 404:
raise
op_start = time()
conn = get_conn(self.conf)
conn.put_container(snapshot['origin'])
logger.warning("failed to retrieve manifest;"
" first time backup for this volume?")
# TODO: write the block_size on the manifest at create?
block_count, remainder = divmod(snapshot['size'], BLOCK_SIZE)
if remainder:
block_count += 1
# initial backup is the only time the we need to worry about
# creating a new manifest for the worker
worker = Worker(snapshot['origin'],
conf=self.conf,
manifest=Manifest.blank(block_count),
stats_path=job_stats_path)
def test_create_first_backup_for_new_volume(self):
h = backup.BackupHelper(self.conf)
def callback():
callback.ran = True
snapshot = {
'id': 'bak1',
'timestamp': 1.0,
}
snapshot['path'] = os.path.join(self.scratch, 'bak1')
snapshot['origin'] = 'vol1'
snapshot['size'] = 4 * 1024 * 1024
with open(snapshot['path'], 'w') as f:
f.write('\x00' * snapshot['size'])
backup_id = 'backup1'
with mock_spawn() as j:
h.create(snapshot, backup_id, callback)
j.run_job()
self.assert_(callback.ran)
conn = get_conn(self.conf)
_headers, raw_json_string = conn.get_object('vol1',
'manifest', newest=True)
m = Manifest.loads(raw_json_string)
self.assertEquals(m.block_count, 1)
self.assertEquals(m.backups['backup1'], 1.0)
self.assertEquals(m.history, [1.0])
self.assert_(isinstance(m[m.history[0]], list))
stats_path = h._stats_file('vol1')
self.assertFalse(os.path.exists(stats_path))
def test_create_first_backup_for_new_volume(self):
h = backup.BackupHelper(self.conf)
def callback():
callback.ran = True
snapshot = {
'id': 'bak1',
'timestamp': 1.0,
}
snapshot['path'] = os.path.join(self.scratch, 'bak1')
snapshot['origin'] = 'vol1'
snapshot['size'] = 4 * 1024 * 1024
with open(snapshot['path'], 'w') as f:
f.write('\x00' * snapshot['size'])
backup_id = 'backup1'
with mock_spawn() as j:
h.create(snapshot, backup_id, callback)
j.run_job()
self.assert_(callback.ran)
conn = get_conn(self.conf)
_headers, raw_json_string = conn.get_object('vol1',
'manifest', newest=True)
m = Manifest.loads(raw_json_string)
self.assertEquals(m.block_count, 1)
self.assertEquals(m.backups['backup1'], 1.0)
self.assertEquals(m.history, [1.0])
self.assert_(isinstance(m[m.history[0]], list))
stats_path = h._stats_file('vol1')
self.assertFalse(os.path.exists(stats_path))
def test_audit(self):
manifest = Manifest.blank(2)
worker = Worker('foo',
LunrConfig({
'backup': {
'client': 'memory'
},
'storage': {
'run_dir': self.scratch
}
}),
manifest=manifest)
conn = worker.conn
conn.put_container('foo')
backup = manifest.create_backup('bak1')
backup[0] = worker.empty_block_hash
conn.put_object('foo', backup[0], 'zeroes')
backup[1] = 'some_block_hash'
conn.put_object('foo', backup[1], ' more stuff')
save_manifest(manifest, conn, worker.id, worker._lock_path())
# Add some non referenced blocks.
conn.put_object('foo', 'stuff1', 'unreferenced stuff1')
conn.put_object('foo', 'stuff2', 'unreferenced stuff2')
conn.put_object('foo', 'stuff3', 'unreferenced stuff3')
_headers, original_list = conn.get_container('foo')
# Manifest, 2 blocks, 3 stuffs.
self.assertEquals(len(original_list), 6)
worker.audit()
_headers, new_list = conn.get_container('foo')
# Manifest, 2 blocks.
self.assertEquals(len(new_list), 3)
def save(self, snapshot, backup_id, cinder):
job_stats_path = self._stats_file(snapshot['origin'])
logger.rename('lunr.storage.helper.backup.save')
setproctitle("lunr-save: " + backup_id)
size = snapshot['size'] / 1024 / 1024 / 1024
try:
op_start = time()
worker = Worker(snapshot['origin'], conf=self.conf,
stats_path=job_stats_path)
except exc.ClientException, e:
if e.http_status != 404:
raise
op_start = time()
conn = get_conn(self.conf)
conn.put_container(snapshot['origin'])
logger.warning("failed to retrieve manifest;"
" first time backup for this volume?")
# TODO: write the block_size on the manifest at create?
block_count, remainder = divmod(snapshot['size'], BLOCK_SIZE)
if remainder:
block_count += 1
# initial backup is the only time the we need to worry about
# creating a new manifest for the worker
worker = Worker(snapshot['origin'], conf=self.conf,
manifest=Manifest.blank(block_count),
stats_path=job_stats_path)
def audit(self, volume):
logger.rename("lunr.storage.helper.backup.audit")
setproctitle("lunr-audit: " + volume["id"])
try:
op_start = time()
worker = Worker(volume["id"], self.conf)
except exc.ClientException, e:
if e.http_status != 404:
raise
op_start = time()
conn = get_conn(self.conf)
conn.put_container(volume["id"])
logger.warning("failed to retrieve manifest;" " auditing volume with no backups")
# creating a blank manifest for the worker
worker = Worker(volume["id"], conf=self.conf, manifest=Manifest.blank(0))
def test_prune_missing_backup_id(self):
h = backup.BackupHelper(self.conf)
volume = {'id': 'vol1', 'size': 1}
existing_backup_id = 'backup1'
missing_backup_id = 'something_else'
m = Manifest.blank(volume['size'])
b = m.create_backup(existing_backup_id, timestamp=1.0)
for i in range(volume['size']):
b[i] = '00'
conn = get_conn(self.conf)
conn.put_container('vol1')
conn.put_object('vol1', '00', 'asdf')
conn.put_object('vol1', 'manifest', m.dumps())
h.prune(volume, missing_backup_id)
# Shouldn't blow up on missing backup_id.
self.assert_(True)
def test_prune_missing_backup_id(self):
h = backup.BackupHelper(self.conf)
volume = {'id': 'vol1', 'size': 1}
existing_backup_id = 'backup1'
missing_backup_id = 'something_else'
m = Manifest.blank(volume['size'])
b = m.create_backup(existing_backup_id, timestamp=1.0)
for i in range(volume['size']):
b[i] = '00'
conn = get_conn(self.conf)
conn.put_container('vol1')
conn.put_object('vol1', '00', 'asdf')
conn.put_object('vol1', 'manifest', m.dumps())
h.prune(volume, missing_backup_id)
# Shouldn't blow up on missing backup_id.
self.assert_(True)
def audit(self, volume):
logger.rename('lunr.storage.helper.backup.audit')
setproctitle("lunr-audit: " + volume['id'])
try:
op_start = time()
worker = Worker(volume['id'], self.conf)
except exc.ClientException, e:
if e.http_status != 404:
raise
op_start = time()
conn = get_conn(self.conf)
conn.put_container(volume['id'])
logger.warning("failed to retrieve manifest;"
" auditing volume with no backups")
# creating a blank manifest for the worker
worker = Worker(volume['id'],
conf=self.conf,
manifest=Manifest.blank(0))
def test_save_stats(self):
manifest = Manifest.blank(2)
stats_path = os.path.join(self.scratch, 'statsfile')
worker = Worker('foo',
LunrConfig({
'backup': {'client': 'memory'},
'storage': {'run_dir': self.scratch}
}),
manifest=manifest,
stats_path=stats_path)
conn = worker.conn
conn.put_container('foo')
worker.save('/dev/zero', 'backup_id', timestamp=1)
try:
with open(stats_path) as f:
json.loads(f.read())
except ValueError:
self.fail("stats path does not contain valid json")
def test_delete_with_missing_blocks(self):
stats_path = os.path.join(self.scratch, 'stats')
manifest = Manifest.blank(2)
worker = Worker('foo',
LunrConfig({
'backup': {
'client': 'memory'
},
'storage': {
'run_dir': self.scratch
}
}),
manifest=manifest)
conn = worker.conn
conn.put_container('foo')
backup = manifest.create_backup('bak1')
backup[0] = worker.empty_block_hash
backup[1] = 'some_random_block_that_isnt_uploaded'
save_manifest(manifest, conn, worker.id, worker._lock_path())
obj = conn.get_object('foo', 'manifest', newest=True)
self.assertRaises(ClientException,
conn.get_object,
'foo',
backup[0],
newest=True)
self.assertRaises(ClientException,
conn.get_object,
'foo',
backup[1],
newest=True)
# Shouldn't blow up on 404.
worker.delete('bak1')
# Manifest should still be nicely deleted.
self.assertRaises(ClientException,
conn.get_object,
'foo',
'manifest',
newest=True)
def test_save_stats(self):
manifest = Manifest.blank(2)
stats_path = os.path.join(self.scratch, 'statsfile')
worker = Worker('foo',
LunrConfig({
'backup': {
'client': 'memory'
},
'storage': {
'run_dir': self.scratch
}
}),
manifest=manifest,
stats_path=stats_path)
conn = worker.conn
conn.put_container('foo')
worker.save('/dev/zero', 'backup_id', timestamp=1)
try:
with open(stats_path) as f:
json.loads(f.read())
except ValueError:
self.fail("stats path does not contain valid json")