def test_delete_with_missing_blocks(self): stats_path = os.path.join(self.scratch, 'stats') manifest = Manifest.blank(2) worker = Worker('foo', LunrConfig({ 'backup': {'client': 'memory'}, 'storage': {'run_dir': self.scratch} }), manifest=manifest) conn = worker.conn conn.put_container('foo') backup = manifest.create_backup('bak1') backup[0] = worker.empty_block_hash backup[1] = 'some_random_block_that_isnt_uploaded' save_manifest(manifest, conn, worker.id, worker._lock_path()) obj = conn.get_object('foo', 'manifest', newest=True) self.assertRaises(ClientException, conn.get_object, 'foo', backup[0], newest=True) self.assertRaises(ClientException, conn.get_object, 'foo', backup[1], newest=True) # Shouldn't blow up on 404. worker.delete('bak1') # Manifest should still be nicely deleted. self.assertRaises(ClientException, conn.get_object, 'foo', 'manifest', newest=True)
def test_audit(self): manifest = Manifest.blank(2) worker = Worker('foo', LunrConfig({ 'backup': {'client': 'memory'}, 'storage': {'run_dir': self.scratch} }), manifest=manifest) conn = worker.conn conn.put_container('foo') backup = manifest.create_backup('bak1') backup[0] = worker.empty_block_hash conn.put_object('foo', backup[0], 'zeroes') backup[1] = 'some_block_hash' conn.put_object('foo', backup[1], ' more stuff') save_manifest(manifest, conn, worker.id, worker._lock_path()) # Add some non referenced blocks. conn.put_object('foo', 'stuff1', 'unreferenced stuff1') conn.put_object('foo', 'stuff2', 'unreferenced stuff2') conn.put_object('foo', 'stuff3', 'unreferenced stuff3') _headers, original_list = conn.get_container('foo') # Manifest, 2 blocks, 3 stuffs. self.assertEquals(len(original_list), 6) worker.audit() _headers, new_list = conn.get_container('foo') # Manifest, 2 blocks. self.assertEquals(len(new_list), 3)
def save(self, snapshot, backup_id, cinder): job_stats_path = self._stats_file(snapshot['origin']) logger.rename('lunr.storage.helper.backup.save') setproctitle("lunr-save: " + backup_id) size = snapshot['size'] / 1024 / 1024 / 1024 try: op_start = time() worker = Worker(snapshot['origin'], conf=self.conf, stats_path=job_stats_path) except exc.ClientException, e: if e.http_status != 404: raise op_start = time() conn = get_conn(self.conf) conn.put_container(snapshot['origin']) logger.warning("failed to retrieve manifest;" " first time backup for this volume?") # TODO: write the block_size on the manifest at create? block_count, remainder = divmod(snapshot['size'], BLOCK_SIZE) if remainder: block_count += 1 # initial backup is the only time the we need to worry about # creating a new manifest for the worker worker = Worker(snapshot['origin'], conf=self.conf, manifest=Manifest.blank(block_count), stats_path=job_stats_path)
def test_audit(self): manifest = Manifest.blank(2) worker = Worker('foo', LunrConfig({ 'backup': { 'client': 'memory' }, 'storage': { 'run_dir': self.scratch } }), manifest=manifest) conn = worker.conn conn.put_container('foo') backup = manifest.create_backup('bak1') backup[0] = worker.empty_block_hash conn.put_object('foo', backup[0], 'zeroes') backup[1] = 'some_block_hash' conn.put_object('foo', backup[1], ' more stuff') save_manifest(manifest, conn, worker.id, worker._lock_path()) # Add some non referenced blocks. conn.put_object('foo', 'stuff1', 'unreferenced stuff1') conn.put_object('foo', 'stuff2', 'unreferenced stuff2') conn.put_object('foo', 'stuff3', 'unreferenced stuff3') _headers, original_list = conn.get_container('foo') # Manifest, 2 blocks, 3 stuffs. self.assertEquals(len(original_list), 6) worker.audit() _headers, new_list = conn.get_container('foo') # Manifest, 2 blocks. self.assertEquals(len(new_list), 3)
def audit(self, volume): logger.rename("lunr.storage.helper.backup.audit") setproctitle("lunr-audit: " + volume["id"]) try: op_start = time() worker = Worker(volume["id"], self.conf) except exc.ClientException, e: if e.http_status != 404: raise op_start = time() conn = get_conn(self.conf) conn.put_container(volume["id"]) logger.warning("failed to retrieve manifest;" " auditing volume with no backups") # creating a blank manifest for the worker worker = Worker(volume["id"], conf=self.conf, manifest=Manifest.blank(0))
def test_prune_missing_backup_id(self): h = backup.BackupHelper(self.conf) volume = {'id': 'vol1', 'size': 1} existing_backup_id = 'backup1' missing_backup_id = 'something_else' m = Manifest.blank(volume['size']) b = m.create_backup(existing_backup_id, timestamp=1.0) for i in range(volume['size']): b[i] = '00' conn = get_conn(self.conf) conn.put_container('vol1') conn.put_object('vol1', '00', 'asdf') conn.put_object('vol1', 'manifest', m.dumps()) h.prune(volume, missing_backup_id) # Shouldn't blow up on missing backup_id. self.assert_(True)
def audit(self, volume): logger.rename('lunr.storage.helper.backup.audit') setproctitle("lunr-audit: " + volume['id']) try: op_start = time() worker = Worker(volume['id'], self.conf) except exc.ClientException, e: if e.http_status != 404: raise op_start = time() conn = get_conn(self.conf) conn.put_container(volume['id']) logger.warning("failed to retrieve manifest;" " auditing volume with no backups") # creating a blank manifest for the worker worker = Worker(volume['id'], conf=self.conf, manifest=Manifest.blank(0))
def test_save_stats(self): manifest = Manifest.blank(2) stats_path = os.path.join(self.scratch, 'statsfile') worker = Worker('foo', LunrConfig({ 'backup': {'client': 'memory'}, 'storage': {'run_dir': self.scratch} }), manifest=manifest, stats_path=stats_path) conn = worker.conn conn.put_container('foo') worker.save('/dev/zero', 'backup_id', timestamp=1) try: with open(stats_path) as f: json.loads(f.read()) except ValueError: self.fail("stats path does not contain valid json")
def test_delete_with_missing_blocks(self): stats_path = os.path.join(self.scratch, 'stats') manifest = Manifest.blank(2) worker = Worker('foo', LunrConfig({ 'backup': { 'client': 'memory' }, 'storage': { 'run_dir': self.scratch } }), manifest=manifest) conn = worker.conn conn.put_container('foo') backup = manifest.create_backup('bak1') backup[0] = worker.empty_block_hash backup[1] = 'some_random_block_that_isnt_uploaded' save_manifest(manifest, conn, worker.id, worker._lock_path()) obj = conn.get_object('foo', 'manifest', newest=True) self.assertRaises(ClientException, conn.get_object, 'foo', backup[0], newest=True) self.assertRaises(ClientException, conn.get_object, 'foo', backup[1], newest=True) # Shouldn't blow up on 404. worker.delete('bak1') # Manifest should still be nicely deleted. self.assertRaises(ClientException, conn.get_object, 'foo', 'manifest', newest=True)
def test_save_stats(self): manifest = Manifest.blank(2) stats_path = os.path.join(self.scratch, 'statsfile') worker = Worker('foo', LunrConfig({ 'backup': { 'client': 'memory' }, 'storage': { 'run_dir': self.scratch } }), manifest=manifest, stats_path=stats_path) conn = worker.conn conn.put_container('foo') worker.save('/dev/zero', 'backup_id', timestamp=1) try: with open(stats_path) as f: json.loads(f.read()) except ValueError: self.fail("stats path does not contain valid json")