def __str__(self): """ format plugin info """ from textwrap import dedent from holland.core.util.fmt import format_bytes, format_datetime return ( dedent( """ Backup: %s start-time: %s stop-time: %s estimated-size: %s on-disk-size: %s """ ).strip() % ( self.name, format_datetime(self.config.lookup("holland:backup.start-time")), format_datetime(self.config.lookup("holland:backup.stop-time")), format_bytes(self.config.lookup("holland:backup.estimated-size")), format_bytes(self.config.lookup("holland:backup.on-disk-size")), ) )
def check_available_space(self, plugin, dry_run=False): estimated_bytes_required = plugin.estimate_backup_size() LOG.info("Estimated Backup Size: %s", format_bytes(estimated_bytes_required)) config = plugin.config['holland:backup'] adjustment_factor = config['estimated-size-factor'] adjusted_bytes_required = (estimated_bytes_required*adjustment_factor) if adjusted_bytes_required != estimated_bytes_required: LOG.info("Adjusting estimated size by %.2f to %s", adjustment_factor, format_bytes(adjusted_bytes_required)) available_bytes = disk_free(self.spool.path) if available_bytes <= adjusted_bytes_required: msg = ("Insufficient Disk Space. %s required, " "but only %s available on %s") % ( format_bytes(adjusted_bytes_required), format_bytes(available_bytes), self.spool.path) if dry_run: LOG.error(msg) LOG.info("Note: This is a dry-run and this " "space may be available during a normal " "backup depending on your purge-policy " "configuration.") else: raise BackupError(msg) return estimated_bytes_required
def check_available_space(self, plugin, spool_entry, dry_run=False): available_bytes = disk_free(spool_entry.path) estimated_bytes_required = float(plugin.estimate_backup_size()) LOG.info("Estimated Backup Size: %s", format_bytes(estimated_bytes_required)) config = plugin.config['holland:backup'] adjustment_factor = float(config['estimated-size-factor']) adjusted_bytes_required = (estimated_bytes_required*adjustment_factor) if adjusted_bytes_required != estimated_bytes_required: LOG.info("Adjusting estimated size by %.2f to %s", adjustment_factor, format_bytes(adjusted_bytes_required)) if available_bytes <= adjusted_bytes_required: if not (config['purge-on-demand'] and self.free_required_space(spool_entry.backupset, adjusted_bytes_required, dry_run)): msg = ("Insufficient Disk Space. %s required, " "but only %s available on %s") % ( format_bytes(adjusted_bytes_required), format_bytes(available_bytes), self.spool.path) LOG.error(msg) if not dry_run: raise BackupError(msg) return float(estimated_bytes_required)
def _formatted_config(self): cfg = dict(self.config["holland:backup"]) cfg["stop-time"] = format_datetime(cfg["stop-time"]) cfg["start-time"] = format_datetime(cfg["start-time"]) cfg["estimated-size"] = format_bytes(cfg["estimated-size"]) cfg["on-disk-size"] = format_bytes(cfg["on-disk-size"]) return cfg
def check_available_space(self, plugin, spool_entry, dry_run=False): available_bytes = disk_free(spool_entry.path) estimated_bytes_required = plugin.estimate_backup_size() LOG.info("Estimated Backup Size: %s", format_bytes(estimated_bytes_required)) config = plugin.config['holland:backup'] adjustment_factor = config['estimated-size-factor'] adjusted_bytes_required = (estimated_bytes_required*adjustment_factor) if adjusted_bytes_required != estimated_bytes_required: LOG.info("Adjusting estimated size by %.2f to %s", adjustment_factor, format_bytes(adjusted_bytes_required)) if available_bytes <= adjusted_bytes_required: if not (config['purge-on-demand'] and self.free_required_space(spool_entry.backupset, adjusted_bytes_required, dry_run)): msg = ("Insufficient Disk Space. %s required, " "but only %s available on %s") % ( format_bytes(adjusted_bytes_required), format_bytes(available_bytes), self.spool.path) LOG.error(msg) if not dry_run: raise BackupError(msg) return estimated_bytes_required
def _formatted_config(self): from holland.core.util.fmt import format_bytes, format_datetime cfg = dict(self.config['holland:backup']) cfg['stop-time'] = format_datetime(cfg['stop-time']) cfg['start-time'] = format_datetime(cfg['start-time']) cfg['estimated-size'] = format_bytes(cfg['estimated-size']) cfg['on-disk-size'] = format_bytes(cfg['on-disk-size']) return cfg
def report_low_space(event, entry): total_space = disk_capacity(entry.path) free_space = disk_free(entry.path) if free_space < 0.10 * total_space: LOG.warning("Extremely low free space on %s's filesystem (%s).", entry.path, getmount(entry.path)) LOG.warning("%s of %s [%.2f%%] remaining", format_bytes(free_space), format_bytes(total_space), (float(free_space) / total_space) * 100)
def verify_space(required_space, target_directory): available_space = disk_free(_find_existing_parent(target_directory)) if required_space >= available_space: LOGGER.error("Insufficient Disk Space. Required: %s Available: %s", format_bytes(required_space), format_bytes(available_space)) raise BackupError("%s required but only %s available on %s" % \ (format_bytes(required_space), format_bytes(available_space), target_directory))
def historic_required_space(self, plugin, spool_entry, estimated_bytes_required): """ Use size reported in 'newest' backup to predict backup size If this fails return a value less than zero and use the estimated-size-factor """ config = plugin.config["holland:backup"] if not config["historic-size"]: return -1.0 historic_size_factor = config["historic-size-factor"] old_backup_config = os.path.join(self.spool.path, spool_entry.backupset, "newest") if not os.path.exists(old_backup_config): LOG.debug("Missing backup.conf from last backup") return -1.0 old_backup = Backup(old_backup_config, spool_entry.backupset, "newest") old_backup.load_config() if ( old_backup.config["holland:backup"]["estimated-size-factor"] and old_backup.config["holland:backup"]["on-disk-size"] ): size_required = old_backup.config["holland:backup"]["on-disk-size"] old_estimate = old_backup.config["holland:backup"]["estimated-size"] else: LOG.debug( "The last backup's configuration was missing the \ ['holland:backup']['on-disk-size'] or ['holland:backup']['estimated-size']" ) return -1.0 LOG.info( "Using Historic Space Estimate: Checking for information in %s", old_backup.config.filename, ) LOG.info("Last backup used %s", format_bytes(size_required)) if estimated_bytes_required > (old_estimate * historic_size_factor): LOG.warning( "The new backup estimate is at least %s times the size of " "the current estimate: %s > (%s * %s). Default back" " to 'estimated-size-factor'", historic_size_factor, format_bytes(estimated_bytes_required), format_bytes(old_estimate), historic_size_factor, ) return -1.0 LOG.debug( "The old and new backup estimate are roughly the same size, " "use old backup size for new size estimate" ) return size_required * float(config["historic-estimated-size-factor"])
def report_low_space(event, entry): total_space = disk_capacity(entry.path) free_space = disk_free(entry.path) if free_space < 0.10 * total_space: LOG.warning("Extremely low free space on %s's filesystem (%s).", entry.path, getmount(entry.path)) LOG.warning( "%s of %s [%.2f%%] remaining", format_bytes(free_space), format_bytes(total_space), (float(free_space) / total_space) * 100, )
def historic_required_space(self, plugin, spool_entry, estimated_bytes_required): """ Use size reported in 'newest' backup to predict backup size If this fails return a value less than zero and use the estimated-size-factor """ config = plugin.config["holland:backup"] if not config["historic-size"]: return -1.0 historic_size_factor = config["historic-size-factor"] old_backup_config = os.path.join(self.spool.path, spool_entry.backupset, "newest") if not os.path.exists(old_backup_config): LOG.debug("Missing backup.conf from last backup") return -1.0 old_backup = Backup(old_backup_config, spool_entry.backupset, "newest") old_backup.load_config() if ( old_backup.config["holland:backup"]["estimated-size-factor"] and old_backup.config["holland:backup"]["on-disk-size"] ): size_required = old_backup.config["holland:backup"]["on-disk-size"] old_estimate = old_backup.config["holland:backup"]["estimated-size"] else: LOG.debug( "The last backup's configuration was missing the \ ['holland:backup']['on-disk-size'] or ['holland:backup']['estimated-size']" ) return -1.0 LOG.info( "Using Historic Space Estimate: Checking for information in %s", old_backup.config.filename, ) LOG.info("Last backup used %s", format_bytes(size_required)) if estimated_bytes_required > (old_estimate * historic_size_factor): LOG.warning( "The new backup estimate is at least %s times the size of " "the currnet estimate: %s > (%s * %s). Default back" " to 'estimated-size-factor'", historic_size_factor, format_bytes(estimated_bytes_required), format_bytes(old_estimate), historic_size_factor, ) return -1.0 LOG.debug( "The old and new backup estimate are roughly the same size, " "use old backup size for new size estimate" ) return size_required * float(config["historic-estimated-size-factor"])
def build_snapshot(config, logical_volume, suppress_tmpdir=False): """Create a snapshot process for running through the various steps of creating, mounting, unmounting and removing a snapshot """ snapshot_name = config['snapshot-name'] or \ logical_volume.lv_name + '_snapshot' extent_size = int(logical_volume.vg_extent_size) snapshot_size = config['snapshot-size'] if not snapshot_size: snapshot_size = min( int(logical_volume.vg_free_count), (int(logical_volume.lv_size) * 0.2) / extent_size, (15 * 1024**3) / extent_size, ) LOG.info("Auto-sizing snapshot-size to %s (%d extents)", format_bytes(snapshot_size * extent_size), snapshot_size) if snapshot_size < 1: raise BackupError( "Insufficient free extents on %s " "to create snapshot (free extents = %s)" % (logical_volume.device_name(), logical_volume.vg_free_count)) else: try: _snapshot_size = snapshot_size snapshot_size = parse_bytes(snapshot_size) / extent_size LOG.info( "Using requested snapshot-size %s " "rounded by extent-size %s to %s.", _snapshot_size, format_bytes(extent_size), format_bytes(snapshot_size * extent_size)) if snapshot_size < 1: raise BackupError("Requested snapshot-size (%s) is " "less than 1 extent" % _snapshot_size) if snapshot_size > int(logical_volume.vg_free_count): LOG.info( "Snapshot size requested %s, but only %s available.", config['snapshot-size'], format_bytes(int(logical_volume.vg_free_count) * extent_size, precision=4)) LOG.info( "Truncating snapshot-size to %d extents (%s)", int(logical_volume.vg_free_count), format_bytes(int(logical_volume.vg_free_count) * extent_size, precision=4)) snapshot_size = int(logical_volume.vg_free_count) except ValueError, exc: raise BackupError("Problem parsing snapshot-size %s" % exc)
def check_available_space(self, plugin, spool_entry, dry_run=False): """ calculate available space before performing backup """ available_bytes = disk_free(spool_entry.path) estimated_bytes_required = float(plugin.estimate_backup_size()) spool_entry.config["holland:backup"]["estimated-size"] = estimated_bytes_required LOG.info("Estimated Backup Size: %s", format_bytes(estimated_bytes_required)) adjusted_bytes_required = self.historic_required_space( plugin, spool_entry, estimated_bytes_required ) config = plugin.config["holland:backup"] if adjusted_bytes_required < 0: adjustment_factor = float(config["estimated-size-factor"]) adjusted_bytes_required = estimated_bytes_required * adjustment_factor if adjusted_bytes_required != estimated_bytes_required: LOG.info( "Adjusting estimated size by %.2f to %s", adjustment_factor, format_bytes(adjusted_bytes_required), ) else: adjustment_factor = float(config["historic-estimated-size-factor"]) LOG.info( "Adjusting estimated size to last backup total * %s: %s", adjustment_factor, format_bytes(adjusted_bytes_required), ) if available_bytes <= adjusted_bytes_required: if not ( config["purge-on-demand"] and self.free_required_space( spool_entry.backupset, adjusted_bytes_required, dry_run ) ): msg = ("Insufficient Disk Space. %s required, " "but only %s available on %s") % ( format_bytes(adjusted_bytes_required), format_bytes(available_bytes), self.spool.path, ) LOG.error(msg) if not dry_run: raise BackupError(msg) return float(estimated_bytes_required)
def legacy_get_db_size(dbname, connection): cursor = connection.cursor() cursor.execute('SELECT SUM(relpages*8192) FROM pg_class') size = int(cursor.fetchone()[0]) LOG.info("DB %s size %s", dbname, format_bytes(size)) cursor.close() return size
def log_final_snapshot_size(event, snapshot): """Log the final size of the snapshot before it is removed""" snapshot.reload() snap_percent = float(snapshot.snap_percent)/100 snap_size = float(snapshot.lv_size) LOG.info("Final LVM snapshot size for %s is %s", snapshot.device_name(), format_bytes(snap_size*snap_percent))
def log_final_snapshot_size(event, snapshot): """Log the final size of the snapshot before it is removed""" snapshot.reload() snap_percent = float(snapshot.snap_percent) / 100 snap_size = float(snapshot.lv_size) LOG.info("Final LVM snapshot size for %s is %s", snapshot.device_name(), format_bytes(snap_size * snap_percent))
def build_snapshot(config, logical_volume): """Create a snapshot process for running through the various steps of creating, mounting, unmounting and removing a snapshot """ snapshot_name = config["snapshot-name"] or logical_volume.lv_name + "_snapshot" extent_size = int(logical_volume.vg_extent_size) snapshot_size = config["snapshot-size"] if not snapshot_size: snapshot_size = min( int(logical_volume.vg_free_count), (int(logical_volume.lv_size) * 0.2) / extent_size, (15 * 1024 ** 3) / extent_size, ) LOG.info( "Auto-sizing snapshot-size to %s (%d extents)", format_bytes(snapshot_size * extent_size), snapshot_size ) if snapshot_size < 1: raise BackupError( "Insufficient free extents on %s " "to create snapshot (free extents = %s)" % (logical_volume.device_name(), logical_volume.vg_free_count) ) else: try: _snapshot_size = snapshot_size snapshot_size = parse_bytes(snapshot_size) / extent_size LOG.info( "Using requested snapshot-size %s " "rounded by extent-size %s to %s.", _snapshot_size, format_bytes(extent_size), format_bytes(snapshot_size * extent_size), ) if snapshot_size < 1: raise BackupError("Requested snapshot-size (%s) is " "less than 1 extent" % _snapshot_size) if snapshot_size > int(logical_volume.vg_free_count): LOG.info( "Snapshot size requested %s, but only %s available.", config["snapshot-size"], format_bytes(int(logical_volume.vg_free_count) * extent_size, precision=4), ) LOG.info( "Truncating snapshot-size to %d extents (%s)", int(logical_volume.vg_free_count), format_bytes(int(logical_volume.vg_free_count) * extent_size, precision=4), ) snapshot_size = int(logical_volume.vg_free_count) except ValueError, exc: raise BackupError("Problem parsing snapshot-size %s" % exc)
def purge_backupset(backupset, force=False, all_backups=False): """Purge a whole backupset either entirely or per the configured retention count :param backupset: Backupset object to purge :param force: Force the purge - this is not a dry-run :param all_backupsets: purge all backups regardless of configured retention count """ if all_backups: retention_count = 0 else: try: config = HOLLANDCFG.backupset(backupset.name) config.validate_config(CONFIGSPEC, suppress_warnings=True) except (IOError, ConfigError) as exc: LOG.error("Failed to load backupset '%s': %s", backupset.name, exc) LOG.error("Aborting, because I could not tell how many backups to " "preserve.") LOG.error( "You can still purge the backupset by using the --all " "option or specifying specific backups to purge" ) else: retention_count = config["holland:backup"]["backups-to-keep"] LOG.info("Evaluating purge for backupset %s", backupset.name) LOG.info("Retaining up to %d backup%s", retention_count, "s"[0 : bool(retention_count)]) backups = [] size = 0 backup_list = backupset.list_backups(reverse=True) for backup in itertools.islice(backup_list, retention_count, None): backups.append(backup) config = backup.config["holland:backup"] size += int(config["on-disk-size"]) LOG.info(" %d total backups", len(backup_list)) for backup in backup_list: LOG.info(" * %s", backup.path) LOG.info(" %d backups to keep", len(backup_list) - len(backups)) for backup in backup_list[0 : -len(backups)]: LOG.info(" + %s", backup.path) LOG.info(" %d backups to purge", len(backups)) for backup in backups: LOG.info(" - %s", backup.path) LOG.info(" %s total to purge", format_bytes(size)) if force: count = 0 for backup in backupset.purge(retention_count): count += 1 LOG.info("Purged %s", backup.name) if count == 0: LOG.info("No backups purged.") else: LOG.info("Purged %d backup%s", count, "s"[0 : bool(count)]) else: LOG.info("Skipping purge in dry-run mode.") backupset.update_symlinks()
def purge_backupset(backupset, force=False, all_backups=False): """Purge a whole backupset either entirely or per the configured retention count :param backupset: Backupset object to purge :param force: Force the purge - this is not a dry-run :param all_backupsets: purge all backups regardless of configured retention count """ if all_backups: retention_count = 0 else: try: config = hollandcfg.backupset(backupset.name) config.validate_config(CONFIGSPEC, suppress_warnings=True) except (IOError, ConfigError) as exc: LOG.error("Failed to load backupset '%s': %s", backupset.name, exc) LOG.error("Aborting, because I could not tell how many backups to " "preserve.") LOG.error("You can still purge the backupset by using the --all " "option or specifying specific backups to purge") return 1 retention_count = config['holland:backup']['backups-to-keep'] LOG.info("Evaluating purge for backupset %s", backupset.name) LOG.info("Retaining up to %d backup%s", retention_count, 's'[0:bool(retention_count)]) backups = [] bytes = 0 backup_list = backupset.list_backups(reverse=True) for backup in itertools.islice(backup_list, retention_count, None): backups.append(backup) config = backup.config['holland:backup'] bytes += int(config['on-disk-size']) LOG.info(" %d total backups", len(backup_list)) for backup in backup_list: LOG.info(" * %s", backup.path) LOG.info(" %d backups to keep", len(backup_list) - len(backups)) for backup in backup_list[0:-len(backups)]: LOG.info(" + %s", backup.path) LOG.info(" %d backups to purge", len(backups)) for backup in backups: LOG.info(" - %s", backup.path) LOG.info(" %s total to purge", format_bytes(bytes)) if force: count = 0 for backup in backupset.purge(retention_count): count += 1 LOG.info("Purged %s", backup.name) if count == 0: LOG.info("No backups purged.") else: LOG.info("Purged %d backup%s", count, 's'[0:bool(count)]) else: LOG.info("Skipping purge in dry-run mode.") backupset.update_symlinks()
def get_db_size(dbname, connection): try: cursor = connection.cursor() cursor.execute("SELECT pg_database_size('%s')" % dbname) size = int(cursor.fetchone()[0]) LOG.info("DB %s size %s", dbname, format_bytes(size)) return size except: raise PgError("Could not detmine database size.")
def check_available_space(self, plugin): estimated_bytes_required = plugin.estimate_backup_size() LOG.info("Estimated Backup Size: %s", format_bytes(estimated_bytes_required)) config = plugin.config['holland:backup'] adjustment_factor = config['estimated-size-factor'] adjusted_bytes_required = (estimated_bytes_required*adjustment_factor) if adjusted_bytes_required != estimated_bytes_required: LOG.info("Adjusting estimated size by %.2f to %s", adjustment_factor, format_bytes(adjusted_bytes_required)) available_bytes = disk_free(self.spool.path) if available_bytes <= adjusted_bytes_required: raise BackupError("Insufficient Disk Space. " "%s required, but only %s available on %s" % (format_bytes(adjusted_bytes_required), format_bytes(available_bytes), self.spool.path)) return estimated_bytes_required
def purge_backup(backup, force=False): """Purge a single backup :param backup: Backup object to purge :param force: Force the purge - this is not a dry-run """ if not force: config = backup.config['holland:backup'] LOG.info("Would purge single backup '%s' %s", backup.name, format_bytes(int(config['on-disk-size']))) else: backup.purge() LOG.info("Purged %s", backup.name)
def free_required_space(self, name, required_bytes, dry_run=False): """Attempt to free at least ``required_bytes`` of old backups from a backupset :param name: name of the backupset to free space from :param required_bytes: integer number of bytes required for the backupset path :param dry_run: if true, this will only generate log messages but won't actually free space :returns: bool; True if freed or False otherwise """ LOG.info( "Insufficient disk space for adjusted estimated backup size: %s", format_bytes(required_bytes), ) LOG.info( "purge-on-demand is enabled. Discovering old backups to purge.") available_bytes = disk_free(os.path.join(self.spool.path, name)) to_purge = {} for backup in self.spool.list_backups(name): backup_size = directory_size(backup.path) LOG.info("Found backup '%s': %s", backup.path, format_bytes(backup_size)) available_bytes += backup_size to_purge[backup] = backup_size if available_bytes > required_bytes: break else: LOG.info("Purging would only recover an additional %s", format_bytes(sum(to_purge.values()))) LOG.info( "Only %s total would be available, but the current " "backup requires %s", format_bytes(available_bytes), format_bytes(required_bytes), ) return False purge_bytes = sum(to_purge.values()) LOG.info( "Found %d backups to purge which will recover %s", len(to_purge), format_bytes(purge_bytes), ) for backup in to_purge: if dry_run: LOG.info("Would purge: %s", backup.path) else: LOG.info("Purging: %s", backup.path) backup.purge() LOG.info( "%s now has %s of available space", os.path.join(self.spool.path, name), format_bytes(disk_free(os.path.join(self.spool.path, name))), ) return True
def __str__(self): """ format plugin info """ return ( dedent( """ Backup: %s start-time: %s stop-time: %s estimated-size: %s on-disk-size: %s """ ).strip() % ( self.name, format_datetime(self.config.lookup("holland:backup.start-time")), format_datetime(self.config.lookup("holland:backup.stop-time")), format_bytes(self.config.lookup("holland:backup.estimated-size")), format_bytes(self.config.lookup("holland:backup.on-disk-size")), ) )
def free_required_space(self, name, required_bytes, dry_run=False): """Attempt to free at least ``required_bytes`` of old backups from a backupset :param name: name of the backupset to free space from :param required_bytes: integer number of bytes required for the backupset path :param dry_run: if true, this will only generate log messages but won't actually free space :returns: bool; True if freed or False otherwise """ LOG.info( "Insufficient disk space for adjusted estimated backup size: %s", format_bytes(required_bytes), ) LOG.info("purge-on-demand is enabled. Discovering old backups to purge.") available_bytes = disk_free(os.path.join(self.spool.path, name)) to_purge = {} for backup in self.spool.list_backups(name): backup_size = directory_size(backup.path) LOG.info("Found backup '%s': %s", backup.path, format_bytes(backup_size)) available_bytes += backup_size to_purge[backup] = backup_size if available_bytes > required_bytes: break else: LOG.info( "Purging would only recover an additional %s", format_bytes(sum(to_purge.values())) ) LOG.info( "Only %s total would be available, but the current " "backup requires %s", format_bytes(available_bytes), format_bytes(required_bytes), ) return False purge_bytes = sum(to_purge.values()) LOG.info( "Found %d backups to purge which will recover %s", len(to_purge), format_bytes(purge_bytes), ) for backup in to_purge: if dry_run: LOG.info("Would purge: %s", backup.path) else: LOG.info("Purging: %s", backup.path) backup.purge() LOG.info( "%s now has %s of available space", os.path.join(self.spool.path, name), format_bytes(disk_free(os.path.join(self.spool.path, name))), ) return True
def _dry_run(self, volume, snapshot, datadir): """Implement dry-run for LVM snapshots. """ LOG.info("* Would snapshot source volume %s/%s as %s/%s (size=%s)", volume.vg_name, volume.lv_name, volume.vg_name, snapshot.name, format_bytes(snapshot.size * int(volume.vg_extent_size))) LOG.info("* Would mount on %s", snapshot.mountpoint or 'generated temporary directory') if getmount(self.target_directory) == getmount(datadir): LOG.error( "Backup directory %s is on the same filesystem as " "the source logical volume %s.", self.target_directory, volume.device_name()) LOG.error("This will result in very poor performance and " "has a high potential for failed backups.") raise BackupError("Improper backup configuration for LVM.")
def _dry_run(target_directory, volume, snapshot, datadir): """Implement dry-run for LVM snapshots. """ LOG.info( volume.vg_name, volume.lv_name, volume.vg_name, snapshot.name, format_bytes(snapshot.size * int(volume.vg_extent_size)), ) LOG.info("* Would mount on %s", snapshot.mountpoint or "generated temporary directory") if getmount(target_directory) == getmount(datadir): LOG.error( "Backup directory %s is on the same filesystem as " "the source logical volume %s.", target_directory, volume.device_name(), ) LOG.error( "This will result in very poor performance and " "has a high potential for failed backups." ) raise BackupError("Improper backup configuration for LVM.")
LOG.info("Starting backup[%s] via plugin %s", spool_entry.name, spool_entry.config['holland:backup']['plugin']) plugin.backup() except KeyboardInterrupt: LOG.warning("Backup aborted by interrupt") spool_entry.config['holland:backup']['failed'] = True except: spool_entry.config['holland:backup']['failed'] = True else: spool_entry.config['holland:backup']['failed'] = False spool_entry.config['holland:backup']['stop-time'] = time.time() if not dry_run and not spool_entry.config['holland:backup']['failed']: final_size = directory_size(spool_entry.path) LOG.info("Final on-disk backup size %s", format_bytes(final_size)) if estimated_size > 0: LOG.info("%.2f%% of estimated size %s", (float(final_size) / estimated_size)*100.0, format_bytes(estimated_size)) spool_entry.config['holland:backup']['on-disk-size'] = final_size spool_entry.flush() start_time = spool_entry.config['holland:backup']['start-time'] stop_time = spool_entry.config['holland:backup']['stop-time'] if spool_entry.config['holland:backup']['failed']: LOG.error("Backup failed after %s", format_interval(stop_time - start_time)) else:
def build_snapshot(config, logical_volume, suppress_tmpdir=False): """Create a snapshot process for running through the various steps of creating, mounting, unmounting and removing a snapshot """ snapshot_name = config["snapshot-name"] or logical_volume.lv_name + "_snapshot" extent_size = int(logical_volume.vg_extent_size) snapshot_size = config["snapshot-size"] if not snapshot_size: snapshot_size = min( int(logical_volume.vg_free_count), (int(logical_volume.lv_size) * 0.2) / extent_size, (15 * 1024 ** 3) / extent_size, ) LOG.info( "Auto-sizing snapshot-size to %s (%d extents)", format_bytes(snapshot_size * extent_size), snapshot_size, ) if snapshot_size < 1: raise BackupError( "Insufficient free extents on %s " "to create snapshot (free extents = %s)" % (logical_volume.device_name(), logical_volume.vg_free_count) ) else: try: _snapshot_size = snapshot_size snapshot_size = parse_bytes(snapshot_size) / extent_size LOG.info( "Using requested snapshot-size %s " "rounded by extent-size %s to %s.", _snapshot_size, format_bytes(extent_size), format_bytes(snapshot_size * extent_size), ) if snapshot_size < 1: raise BackupError( "Requested snapshot-size (%s) is " "less than 1 extent" % _snapshot_size ) if snapshot_size > int(logical_volume.vg_free_count): LOG.info( "Snapshot size requested %s, but only %s available.", config["snapshot-size"], format_bytes(int(logical_volume.vg_free_count) * extent_size, precision=4), ) LOG.info( "Truncating snapshot-size to %d extents (%s)", int(logical_volume.vg_free_count), format_bytes(int(logical_volume.vg_free_count) * extent_size, precision=4), ) snapshot_size = int(logical_volume.vg_free_count) except ValueError as exc: raise BackupError("Problem parsing snapshot-size %s" % exc) mountpoint = config["snapshot-mountpoint"] tempdir = False if not mountpoint: tempdir = True if not suppress_tmpdir: mountpoint = tempfile.mkdtemp() else: try: os.makedirs(mountpoint) LOG.info("Created mountpoint %s", mountpoint) except OSError as exc: # silently ignore if the mountpoint already exists if exc.errno != errno.EEXIST: # pylint: disable=no-member raise BackupError("Failure creating snapshot mountpoint: %s" % str(exc)) snapshot = Snapshot(snapshot_name, int(snapshot_size), mountpoint) if tempdir: snapshot.register("finish", lambda *args, **kwargs: cleanup_tempdir(mountpoint)) return snapshot
LOG.info("Starting backup[%s] via plugin %s", spool_entry.name, spool_entry.config['holland:backup']['plugin']) plugin.backup() except KeyboardInterrupt: LOG.warning("Backup aborted by interrupt") spool_entry.config['holland:backup']['failed'] = True except: spool_entry.config['holland:backup']['failed'] = True else: spool_entry.config['holland:backup']['failed'] = False spool_entry.config['holland:backup']['stop-time'] = time.time() if not dry_run and not spool_entry.config['holland:backup']['failed']: final_size = float(directory_size(spool_entry.path)) LOG.info("Final on-disk backup size %s", format_bytes(final_size)) if estimated_size > 0: LOG.info("%.2f%% of estimated size %s", (final_size / estimated_size)*100.0, format_bytes(estimated_size)) spool_entry.config['holland:backup']['on-disk-size'] = final_size spool_entry.flush() start_time = spool_entry.config['holland:backup']['start-time'] stop_time = spool_entry.config['holland:backup']['stop-time'] if spool_entry.config['holland:backup']['failed']: LOG.error("Backup failed after %s", format_interval(stop_time - start_time)) else:
def backup(self, name, config, dry_run=False): """Run a backup for the named backupset using the provided configuration :param name: name of the backupset :param config: dict-like object providing the backupset configuration :raises: BackupError if a backup fails """ for i in range(MAX_SPOOL_RETRIES): try: spool_entry = self.spool.add_backup(name) break except OSError as exc: if exc.errno != errno.EEXIST: raise BackupError("Failed to create spool: %s" % exc) LOG.debug("Failed to create spool. Retrying in %d seconds.", i + 1) time.sleep(i + 1) else: raise BackupError("Failed to create a new backup directory for %s" % name) spool_entry.config.merge(config) spool_entry.validate_config() if dry_run: # always purge the spool self.register_cb("post-backup", lambda *args, **kwargs: spool_entry.purge()) plugin = load_plugin(name, spool_entry.config, spool_entry.path, dry_run) spool_entry.config["holland:backup"]["start-time"] = time.time() spool_entry.flush() self.apply_cb("before-backup", spool_entry) spool_entry.config["holland:backup"]["failed"] = False try: estimated_size = self.check_available_space(plugin, spool_entry, dry_run) LOG.info( "Starting backup[%s] via plugin %s", spool_entry.name, spool_entry.config["holland:backup"]["plugin"], ) plugin.backup() except KeyboardInterrupt: LOG.warning("Backup aborted by interrupt") spool_entry.config["holland:backup"]["failed"] = True raise except BaseException as ex: LOG.warning(ex) spool_entry.config["holland:backup"]["failed"] = True spool_entry.config["holland:backup"]["stop-time"] = time.time() if not dry_run and not spool_entry.config["holland:backup"]["failed"]: final_size = float(directory_size(spool_entry.path)) LOG.info("Final on-disk backup size %s", format_bytes(final_size)) if estimated_size > 0: LOG.info( "%.2f%% of estimated size %s", (final_size / estimated_size) * 100.0, format_bytes(estimated_size), ) spool_entry.config["holland:backup"]["on-disk-size"] = final_size spool_entry.flush() start_time = spool_entry.config["holland:backup"]["start-time"] stop_time = spool_entry.config["holland:backup"]["stop-time"] if spool_entry.config["holland:backup"]["failed"]: LOG.error("Backup failed after %s", format_interval(stop_time - start_time)) else: LOG.info("Backup completed in %s", format_interval(stop_time - start_time)) if dry_run: spool_entry.purge() if sys.exc_info() != (None, None, None) or spool_entry.config["holland:backup"]["failed"]: LOG.debug("sys.exc_info(): %r", sys.exc_info()) self.apply_cb("failed-backup", spool_entry) raise BackupError("Failed backup: %s" % name) else: self.apply_cb("after-backup", spool_entry)
backup_list = backupset.list_backups(reverse=True) for backup in itertools.islice(backup_list, retention_count, None): backups.append(backup) config = backup.config['holland:backup'] bytes += int(config['on-disk-size']) LOG.info(" %d total backups", len(backup_list)) for backup in backup_list: LOG.info(" * %s", backup.path) LOG.info(" %d backups to keep", len(backup_list) - len(backups)) for backup in backup_list[0:-len(backups)]: LOG.info(" + %s", backup.path) LOG.info(" %d backups to purge", len(backups)) for backup in backups: LOG.info(" - %s", backup.path) LOG.info(" %s total to purge", format_bytes(bytes)) if force: count = 0 for backup in backupset.purge(retention_count): count += 1 LOG.info("Purged %s", backup.name) if count == 0: LOG.info("No backups purged.") else: LOG.info("Purged %d backup%s", count, 's'[0:bool(count)]) else: LOG.info("Skipping purge in dry-run mode.") backupset.update_symlinks() def purge_backup(backup, force=False):
def build_snapshot(config, logical_volume, suppress_tmpdir=False): """Create a snapshot process for running through the various steps of creating, mounting, unmounting and removing a snapshot """ snapshot_name = config['snapshot-name'] or \ logical_volume.lv_name + '_snapshot' extent_size = int(logical_volume.vg_extent_size) snapshot_size = config['snapshot-size'] if not snapshot_size: snapshot_size = min( int(logical_volume.vg_free_count), (int(logical_volume.lv_size) * 0.2) / extent_size, (15 * 1024**3) / extent_size, ) LOG.info("Auto-sizing snapshot-size to %s (%d extents)", format_bytes(snapshot_size * extent_size), snapshot_size) if snapshot_size < 1: raise BackupError( "Insufficient free extents on %s " "to create snapshot (free extents = %s)" % (logical_volume.device_name(), logical_volume.vg_free_count)) else: try: _snapshot_size = snapshot_size snapshot_size = parse_bytes(snapshot_size) / extent_size LOG.info( "Using requested snapshot-size %s " "rounded by extent-size %s to %s.", _snapshot_size, format_bytes(extent_size), format_bytes(snapshot_size * extent_size)) if snapshot_size < 1: raise BackupError("Requested snapshot-size (%s) is " "less than 1 extent" % _snapshot_size) if snapshot_size > int(logical_volume.vg_free_count): LOG.info( "Snapshot size requested %s, but only %s available.", config['snapshot-size'], format_bytes(int(logical_volume.vg_free_count) * extent_size, precision=4)) LOG.info( "Truncating snapshot-size to %d extents (%s)", int(logical_volume.vg_free_count), format_bytes(int(logical_volume.vg_free_count) * extent_size, precision=4)) snapshot_size = int(logical_volume.vg_free_count) except ValueError as exc: raise BackupError("Problem parsing snapshot-size %s" % exc) mountpoint = config['snapshot-mountpoint'] tempdir = False if not mountpoint: tempdir = True if not suppress_tmpdir: mountpoint = tempfile.mkdtemp() else: try: os.makedirs(mountpoint) LOG.info("Created mountpoint %s", mountpoint) except OSError as exc: # silently ignore if the mountpoint already exists if exc.errno != errno.EEXIST: raise BackupError("Failure creating snapshot mountpoint: %s" % str(exc)) snapshot = Snapshot(snapshot_name, int(snapshot_size), mountpoint) if tempdir: snapshot.register('finish', lambda *args, **kwargs: cleanup_tempdir(mountpoint)) return snapshot
dry_run) except Exception, e: LOGGER.debug("Failed to instantiate backup plugin %s: %s", backupset_cfg.lookup('holland:backup.plugin'), e, exc_info=True) raise BackupError("Failed to initialize backup plugin %s: %s" % (backupset_cfg.lookup('holland:backup.plugin'), e)) # Plugin may raise exception due to programming error, be careful estimated_size = plugin.estimate_backup_size() estimate_factor = backup_job.config['holland:backup'][ 'estimated-size-factor'] adjusted_estimate = estimate_factor * estimated_size LOGGER.info("Estimated Backup Size: %s", format_bytes(estimated_size)) if adjusted_estimate != estimated_size: LOGGER.info( "Using estimated-size-factor=%.2f and adjusting estimate to %s", estimate_factor, format_bytes(adjusted_estimate)) # Save the estimated size in the backup.conf backup_job.config['holland:backup']['estimated-size'] = estimated_size try: verify_space(adjusted_estimate, backup_job.path) except BackupError, exc: if not dry_run: raise if not dry_run: LOGGER.info("Purging old backup jobs")
try: plugin = plugincls(backupset_name, backup_job.config, backup_job.path, dry_run) except Exception, e: LOGGER.debug("Failed to instantiate backup plugin %s: %s", backupset_cfg.lookup('holland:backup.plugin'), e, exc_info=True) raise BackupError("Failed to initialize backup plugin %s: %s" % (backupset_cfg.lookup('holland:backup.plugin'), e)) # Plugin may raise exception due to programming error, be careful estimated_size = plugin.estimate_backup_size() estimate_factor = backup_job.config['holland:backup']['estimated-size-factor'] adjusted_estimate = estimate_factor*estimated_size LOGGER.info("Estimated Backup Size: %s", format_bytes(estimated_size) ) if adjusted_estimate != estimated_size: LOGGER.info("Using estimated-size-factor=%.2f and adjusting estimate to %s", estimate_factor, format_bytes(adjusted_estimate) ) # Save the estimated size in the backup.conf backup_job.config['holland:backup']['estimated-size'] = estimated_size try: verify_space(adjusted_estimate, backup_job.path) except BackupError, exc: if not dry_run: raise