def setUp(self): opts = osclients.OpenstackOpts.create_from_env().get_opts_dicts() self.storage = swift.SwiftStorage( osclients.OSClientManager(opts.pop('auth_url'), opts.pop('auth_method', 'password'), **opts ), "freezer_ops-aw1ops1-gerrit0001.aw1.hpcloud.net", "/tmp/", 100, skip_prepare=True ) self.files = [ "tar_metadata_hostname_backup_1000_0", "hostname_backup_1000_0", ] self.increments = [ "tar_metadata_hostname_backup_1000_0", "hostname_backup_1000_0", "tar_metadata_hostname_backup_2000_1", "hostname_backup_2000_1", ] self.cycles_increments = [ "tar_metadata_hostname_backup_1000_0", "hostname_backup_1000_0", "tar_metadata_hostname_backup_2000_1", "hostname_backup_2000_1", "tar_metadata_hostname_backup_3000_0", "hostname_backup_3000_0", "tar_metadata_hostname_backup_4000_1", "hostname_backup_4000_1", ] self.backup = base.Backup(self.storage, "hostname_backup", 1000, tar_meta=True,) self.backup_2 = base.Backup(self.storage, "hostname_backup", 3000, tar_meta=True) self.increment = base.Backup(self.storage, "hostname_backup", 2000, full_backup=self.backup, level=1, tar_meta=True) self.increment_2 = base.Backup(self.storage, "hostname_backup", 4000, full_backup=self.backup_2, level=1, tar_meta=True)
def get_level_zero(self, engine, hostname_backup_name, recent_to_date=None): """ Gets backups by backup_name and hostname :type engine: freezer.engine.engine.BackupEngine :param engine: Search for backups made by specified engine :type hostname_backup_name: str :param hostname_backup_name: Search for backup with specified name :type recent_to_date: int :param recent_to_date: :rtype: list[freezer.storage.base.Backup] :return: dictionary of level zero timestamps with attached storage """ path = self.metadata_path(engine=engine, hostname_backup_name=hostname_backup_name) zeros = [ base.Backup(storage=self, engine=engine, hostname_backup_name=hostname_backup_name, level_zero_timestamp=int(t), timestamp=int(t), level=0) for t in self.listdir(path) ] if recent_to_date: zeros = [ zero for zero in zeros if zero.timestamp >= recent_to_date ] return zeros
def backup(self, backup_resource, hostname_backup_name, no_incremental, max_level, always_level, restart_always_level, queue_size=2): """ Here we now location of all interesting artifacts like metadata Should return stream for storing data. :return: stream """ prev_backup = self.storage.previous_backup( engine=self, hostname_backup_name=hostname_backup_name, no_incremental=no_incremental, max_level=max_level, always_level=always_level, restart_always_level=restart_always_level ) try: tmpdir = tempfile.mkdtemp() except Exception: LOG.error("Unable to create a tmp directory") raise try: engine_meta = utils.path_join(tmpdir, "engine_meta") freezer_meta = utils.path_join(tmpdir, "freezer_meta") if prev_backup: prev_backup.storage.get_file(prev_backup.engine_metadata_path, engine_meta) timestamp = utils.DateTime.now().timestamp level_zero_timestamp = (prev_backup.level_zero_timestamp if prev_backup else timestamp) backup = base.Backup( engine=self, hostname_backup_name=hostname_backup_name, level_zero_timestamp=level_zero_timestamp, timestamp=timestamp, level=(prev_backup.level + 1 if prev_backup else 0) ) input_queue = streaming.RichQueue(queue_size) read_except_queue = queue.Queue() write_except_queue = queue.Queue() read_stream = streaming.QueuedThread( self.backup_stream, input_queue, read_except_queue, kwargs={"backup_resource": backup_resource, "manifest_path": engine_meta}) write_stream = streaming.QueuedThread( self.storage.write_backup, input_queue, write_except_queue, kwargs={"backup": backup}) read_stream.daemon = True write_stream.daemon = True read_stream.start() write_stream.start() read_stream.join() write_stream.join() # queue handling is different from SimpleQueue handling. def handle_except_queue(except_queue): if not except_queue.empty(): while not except_queue.empty(): e = except_queue.get_nowait() LOG.critical('Engine error: {0}'.format(e)) return True else: return False got_exception = None got_exception = (handle_except_queue(read_except_queue) or got_exception) got_exception = (handle_except_queue(write_except_queue) or got_exception) if got_exception: raise engine_exceptions.EngineException( "Engine error. Failed to backup.") with open(freezer_meta, mode='wb') as b_file: b_file.write(json.dumps(self.metadata(backup_resource))) self.storage.put_metadata(engine_meta, freezer_meta, backup) finally: shutil.rmtree(tmpdir)