def fullbackup_main(bin_dir, storages, keep, full, verbose, gzip, backup_blobs, only_blobs, use_rsync, keep_blob_days=0, pre_command='', post_command='', **kwargs): """Main method, gets called by generated bin/fullbackup.""" utils.execute_or_fail(pre_command) if not only_blobs: # Set Full=True for forced full backups. # It was easier to do this here, than mess with # "script_arguments = arguments_template % opts" # in backup.Recipe.install full = True result = repozorunner.fullbackup_main( bin_dir, storages, keep, full, verbose, gzip) if result and backup_blobs: logger.error("Halting execution due to error; not backing up " "blobs.") if not backup_blobs: utils.execute_or_fail(post_command) return for storage in storages: blobdir = storage['blobdir'] if not blobdir: logger.info("No blob dir defined for %s storage" % storage['storage']) continue blob_backup_location = storage['blob_backup_location'] logger.info("Please wait while backing up blobs from %s to %s", blobdir, blob_backup_location) copyblobs.backup_blobs(blobdir, blob_backup_location, full, use_rsync, keep=keep, keep_blob_days=keep_blob_days,) utils.execute_or_fail(post_command)
def snapshot_main(bin_dir, storages, keep, verbose, gzip, backup_blobs, only_blobs, use_rsync, keep_blob_days=0, pre_command='', post_command='', **kwargs): """Main method, gets called by generated bin/snapshotbackup.""" utils.execute_or_fail(pre_command) if not only_blobs: result = repozorunner.snapshot_main( bin_dir, storages, keep, verbose, gzip) if result and backup_blobs: logger.error("Halting execution due to error; not backing up " "blobs.") if not backup_blobs: utils.execute_or_fail(post_command) return for storage in storages: blobdir = storage['blobdir'] if not blobdir: logger.info("No blob dir defined for %s storage" % \ (storage['storage'])) continue blob_snapshot_location = storage['blob_snapshot_location'] logger.info("Please wait while making snapshot of blobs from %s to %s", blobdir, blob_snapshot_location) copyblobs.backup_blobs(blobdir, blob_snapshot_location, full=True, use_rsync=use_rsync, keep=keep, keep_blob_days=keep_blob_days) utils.execute_or_fail(post_command)
def backup_main(bin_dir, storages, keep, full, verbose, gzip, backup_blobs, only_blobs, use_rsync, keep_blob_days=0, pre_command='', post_command='', **kwargs): """Main method, gets called by generated bin/backup.""" utils.check_folders(storages) utils.execute_or_fail(pre_command) if not only_blobs: result = repozorunner.backup_main( bin_dir, storages, keep, full, verbose, gzip) if result: if backup_blobs: logger.error("Halting execution due to error; not backing up " "blobs.") else: logger.error("Halting execution due to error.") sys.exit(1) if not backup_blobs: utils.execute_or_fail(post_command) return for storage in storages: blobdir = storage['blobdir'] if not blobdir: logger.info("No blob dir defined for %s storage" % storage['storage']) continue blob_backup_location = storage['blob_backup_location'] logger.info("Please wait while backing up blobs from %s to %s", blobdir, blob_backup_location) copyblobs.backup_blobs(blobdir, blob_backup_location, full, use_rsync, keep=keep, keep_blob_days=keep_blob_days,) utils.execute_or_fail(post_command)
def backup_main(bin_dir, storages, keep, full, verbose, gzip, backup_blobs, only_blobs, use_rsync, keep_blob_days=0, pre_command='', post_command='', gzip_blob=False, rsync_options='', quick=True, **kwargs): """Main method, gets called by generated bin/backup.""" utils.check_folders(storages, backup_blobs=backup_blobs, only_blobs=only_blobs, backup=True, snapshot=False, zipbackup=False) utils.execute_or_fail(pre_command) if not only_blobs: result = repozorunner.backup_main(bin_dir, storages, keep, full, verbose, gzip, quick) if result: if backup_blobs: logger.error("Halting execution due to error; not backing up " "blobs.") else: logger.error("Halting execution due to error.") sys.exit(1) if not backup_blobs: utils.execute_or_fail(post_command) return for storage in storages: blobdir = storage['blobdir'] if not blobdir: logger.info("No blob dir defined for %s storage" % storage['storage']) continue blob_backup_location = storage['blob_backup_location'] logger.info("Please wait while backing up blobs from %s to %s", blobdir, blob_backup_location) copyblobs.backup_blobs(blobdir, blob_backup_location, full, use_rsync, keep=keep, keep_blob_days=keep_blob_days, gzip_blob=gzip_blob, rsync_options=rsync_options) utils.execute_or_fail(post_command)
def restore_main(bin_dir, storages, verbose, backup_blobs, only_blobs, use_rsync, restore_snapshot=False, pre_command='', post_command='', **kwargs): """Main method, gets called by generated bin/restore.""" date = None if len(sys.argv) > 1: date = sys.argv[1] logger.debug("Argument passed to bin/restore, we assume it is " "a date that we have to pass to repozo: %s.", date) logger.info("Date restriction: restoring state at %s." % date) question = '\n' if not only_blobs: question += "This will replace the filestorage (Data.fs).\n" if backup_blobs: question += "This will replace the blobstorage.\n" question += "Are you sure?" if not utils.ask(question, default=False, exact=True): logger.info("Not restoring.") sys.exit(0) utils.execute_or_fail(pre_command) if not only_blobs: result = repozorunner.restore_main( bin_dir, storages, verbose, date, restore_snapshot) if result and backup_blobs: logger.error("Halting execution due to error; not restoring " "blobs.") sys.exit(1) if not backup_blobs: utils.execute_or_fail(post_command) return for storage in storages: blobdir = storage['blobdir'] if restore_snapshot: blob_backup_location = storage['blob_snapshot_location'] else: blob_backup_location = storage['blob_backup_location'] if not blobdir: logger.info("No blob dir defined for %s storage" % \ (storage['storage'])) continue if not blobdir: logger.error("No blob storage source specified") sys.exit(1) logger.info("Restoring blobs from %s to %s", blob_backup_location, blobdir) copyblobs.restore_blobs(blob_backup_location, blobdir, use_rsync=use_rsync, date=date) utils.execute_or_fail(post_command)
def restore_main(bin_dir, storages, verbose, backup_blobs, only_blobs, use_rsync, restore_snapshot=False, pre_command='', post_command='', gzip_blob=False, alt_restore=False, **kwargs): """Main method, gets called by generated bin/restore.""" if restore_snapshot and alt_restore: logger.error("Cannot use both restore_snapshot and alt_restore.") sys.exit(1) date = None # Try to find a date in the command line arguments for arg in sys.argv: if arg in ('-q', '-n', '--quiet', '--no-prompt'): continue if arg.find('restore') != -1: continue # We can assume this argument is a date date = arg logger.debug("Argument passed to bin/restore, we assume it is " "a date that we have to pass to repozo: %s.", date) logger.info("Date restriction: restoring state at %s." % date) break question = '\n' if not only_blobs: question += "This will replace the filestorage:\n" for storage in storages: question += " %s\n" % storage.get('datafs') if backup_blobs: question += "This will replace the blobstorage:\n" for storage in storages: if storage.get('blobdir'): question += " %s\n" % storage.get('blobdir') question += "Are you sure?" if not kwargs.get('no_prompt'): if not utils.ask(question, default=False, exact=True): logger.info("Not restoring.") sys.exit(0) utils.execute_or_fail(pre_command) if not only_blobs: result = repozorunner.restore_main( bin_dir, storages, verbose, date, restore_snapshot, alt_restore) if result: if backup_blobs: logger.error("Halting execution due to error; not restoring " "blobs.") else: logger.error("Halting execution due to error.") sys.exit(1) if not backup_blobs: utils.execute_or_fail(post_command) return for storage in storages: blobdir = storage['blobdir'] if not blobdir: logger.info("No blob dir defined for %s storage" % storage['storage']) continue if restore_snapshot: blob_backup_location = storage['blob_snapshot_location'] elif alt_restore: blob_backup_location = storage['blob_alt_location'] else: blob_backup_location = storage['blob_backup_location'] if not blob_backup_location: logger.error("No blob storage source specified") sys.exit(1) logger.info("Restoring blobs from %s to %s", blob_backup_location, blobdir) copyblobs.restore_blobs(blob_backup_location, blobdir, use_rsync=use_rsync, date=date, gzip_blob=gzip_blob) utils.execute_or_fail(post_command)
def restore_main( bin_dir, storages, verbose, backup_blobs, only_blobs, use_rsync, restore_snapshot=False, pre_command="", post_command="", archive_blob=False, alt_restore=False, rsync_options="", quick=True, zip_restore=False, blob_timestamps=False, incremental_blobs=False, **kwargs ): """Main method, gets called by generated bin/restore.""" # First run several checks, and get the date that should be restored. date = restore_check( bin_dir, storages, verbose, backup_blobs, only_blobs, use_rsync, restore_snapshot=restore_snapshot, pre_command=pre_command, post_command=post_command, archive_blob=archive_blob, alt_restore=alt_restore, rsync_options=rsync_options, quick=quick, zip_restore=zip_restore, blob_timestamps=blob_timestamps, incremental_blobs=incremental_blobs, **kwargs ) # Checks have passed, now do the real restore. if not only_blobs: result = repozorunner.restore_main( bin_dir, storages, verbose, date, restore_snapshot, alt_restore, zip_restore ) if result: if backup_blobs: logger.error("Halting execution due to error; not restoring " "blobs.") else: logger.error("Halting execution due to error.") sys.exit(1) if not backup_blobs: utils.execute_or_fail(post_command) return for storage in storages: blobdir = storage["blobdir"] if not blobdir: continue blob_backup_location = storage["blob_backup_location"] logger.info("Restoring blobs from %s to %s", blob_backup_location, blobdir) result = copyblobs.restore_blobs( blob_backup_location, blobdir, use_rsync=use_rsync, date=date, archive_blob=archive_blob, rsync_options=rsync_options, timestamps=blob_timestamps, ) if result: logger.error("Halting execution due to error.") sys.exit(1) utils.execute_or_fail(post_command)
def restore_check( bin_dir, storages, verbose, backup_blobs, only_blobs, use_rsync, restore_snapshot=False, pre_command="", post_command="", archive_blob=False, alt_restore=False, rsync_options="", quick=True, zip_restore=False, blob_timestamps=False, **kwargs ): """Method to check that a restore will work. Returns the chosen date, if any. """ explicit_restore_opts = [restore_snapshot, alt_restore, zip_restore] if sum([1 for opt in explicit_restore_opts if opt]) > 1: logger.error( "Must use at most one option of restore_snapshot, " "alt_restore and zip_restore." ) sys.exit(1) # Try to find a date in the command line arguments date = utils.get_date_from_args() if not kwargs.get("no_prompt"): question = "\n" if not only_blobs: question += "This will replace the filestorage:\n" for storage in storages: question += " {0}\n".format(storage.get("datafs")) if backup_blobs: question += "This will replace the blobstorage:\n" for storage in storages: if storage.get("blobdir"): question += " {0}\n".format(storage.get("blobdir")) question += "Are you sure?" if not utils.ask(question, default=False, exact=True): logger.info("Not restoring.") sys.exit(0) utils.execute_or_fail(pre_command) # First run some checks. if not only_blobs: result = repozorunner.restore_main( bin_dir, storages, verbose, date, restore_snapshot, alt_restore, zip_restore, only_check=True, ) if result: logger.error("Halting execution: " "restoring filestorages would fail.") sys.exit(1) if backup_blobs: check_blobs( storages, use_rsync, restore_snapshot=restore_snapshot, archive_blob=archive_blob, alt_restore=alt_restore, rsync_options=rsync_options, zip_restore=zip_restore, blob_timestamps=blob_timestamps, date=date, ) return date
def backup_main( bin_dir, storages, keep, full, verbose, gzip, backup_blobs, only_blobs, use_rsync, keep_blob_days=0, pre_command="", post_command="", archive_blob=False, compress_blob=False, rsync_options="", quick=True, blob_timestamps=False, backup_method=config.STANDARD_BACKUP, incremental_blobs=False, **kwargs ): """Main method, gets called by generated bin/backup.""" if backup_method not in config.BACKUP_METHODS: raise RuntimeError("Unknown backup method {0}.".format(backup_method)) utils.execute_or_fail(pre_command) utils.check_folders( storages, backup_blobs=backup_blobs, only_blobs=only_blobs, backup_method=backup_method, ) if not only_blobs: result = repozorunner.backup_main( bin_dir, storages, keep, full, verbose, gzip, quick, backup_method ) if result: if backup_blobs: logger.error("Halting execution due to error; not backing up blobs.") else: logger.error("Halting execution due to error.") sys.exit(1) if not backup_blobs: utils.execute_or_fail(post_command) return for storage in storages: blobdir = storage["blobdir"] if not blobdir: logger.info("No blob dir defined for %s storage", storage["storage"]) continue blob_backup_location = None if backup_method == config.STANDARD_BACKUP: blob_backup_location = storage["blob_backup_location"] logger.info( "Please wait while backing up blobs from %s to %s", blobdir, blob_backup_location, ) elif backup_method == config.SNAPSHOT_BACKUP: blob_backup_location = storage["blob_snapshot_location"] logger.info( "Please wait while making snapshot of blobs from %s to %s", blobdir, blob_backup_location, ) elif backup_method == config.ZIP_BACKUP: blob_backup_location = storage["blob_zip_location"] logger.info( "Please wait while backing up blobs from %s to %s", blobdir, blob_backup_location, ) if only_blobs: fs_backup_location = None else: fs_backup_location = storage["backup_location"] copyblobs.backup_blobs( blobdir, blob_backup_location, full, use_rsync, keep=keep, keep_blob_days=keep_blob_days, archive_blob=archive_blob, compress_blob=compress_blob, rsync_options=rsync_options, timestamps=blob_timestamps, fs_backup_location=fs_backup_location, incremental_blobs=incremental_blobs, ) utils.execute_or_fail(post_command)
def fullbackup_main(bin_dir, storages, keep, full, verbose, gzip, backup_blobs, only_blobs, use_rsync, keep_blob_days=0, pre_command='', post_command='', gzip_blob=False, rsync_options='', quick=True, **kwargs): """Main method, gets called by generated bin/fullbackup.""" utils.execute_or_fail(pre_command) utils.check_folders(storages, backup_blobs=backup_blobs, only_blobs=only_blobs, backup=True, snapshot=False, zipbackup=False) if not only_blobs: # Set Full=True for forced full backups. # It was easier to do this here, than mess with # "script_arguments = arguments_template % opts" # in backup.Recipe.install full = True result = repozorunner.fullbackup_main(bin_dir, storages, keep, full, verbose, gzip) if result: if backup_blobs: logger.error("Halting execution due to error; not backing up " "blobs.") else: logger.error("Halting execution due to error.") sys.exit(1) if not backup_blobs: utils.execute_or_fail(post_command) return for storage in storages: blobdir = storage['blobdir'] if not blobdir: logger.info("No blob dir defined for %s storage" % storage['storage']) continue blob_backup_location = storage['blob_backup_location'] logger.info("Please wait while backing up blobs from %s to %s", blobdir, blob_backup_location) copyblobs.backup_blobs(blobdir, blob_backup_location, full, use_rsync, keep=keep, keep_blob_days=keep_blob_days, gzip_blob=gzip_blob, rsync_options=rsync_options) utils.execute_or_fail(post_command)
def restore_main(bin_dir, storages, verbose, backup_blobs, only_blobs, use_rsync, restore_snapshot=False, pre_command='', post_command='', gzip_blob=False, alt_restore=False, rsync_options='', quick=True, zip_restore=False, **kwargs): """Main method, gets called by generated bin/restore.""" explicit_restore_opts = [restore_snapshot, alt_restore, zip_restore] if sum([1 for opt in explicit_restore_opts if opt]) > 1: logger.error("Must use at most one option of restore_snapshot, " "alt_restore and zip_restore.") sys.exit(1) date = None # Try to find a date in the command line arguments for arg in sys.argv: if arg in ('-q', '-n', '--quiet', '--no-prompt'): continue if arg.find('restore') != -1: continue # We can assume this argument is a date date = arg logger.debug( "Argument passed to bin/restore, we assume it is " "a date that we have to pass to repozo: %s.", date) logger.info("Date restriction: restoring state at %s." % date) break question = '\n' if not only_blobs: question += "This will replace the filestorage:\n" for storage in storages: question += " %s\n" % storage.get('datafs') if backup_blobs: question += "This will replace the blobstorage:\n" for storage in storages: if storage.get('blobdir'): question += " %s\n" % storage.get('blobdir') question += "Are you sure?" if not kwargs.get('no_prompt'): if not utils.ask(question, default=False, exact=True): logger.info("Not restoring.") sys.exit(0) utils.execute_or_fail(pre_command) if not only_blobs: result = repozorunner.restore_main(bin_dir, storages, verbose, date, restore_snapshot, alt_restore, zip_restore) if result: if backup_blobs: logger.error("Halting execution due to error; not restoring " "blobs.") else: logger.error("Halting execution due to error.") sys.exit(1) if not backup_blobs: utils.execute_or_fail(post_command) return for storage in storages: blobdir = storage['blobdir'] if not blobdir: logger.info("No blob dir defined for %s storage" % storage['storage']) continue if restore_snapshot: blob_backup_location = storage['blob_snapshot_location'] elif alt_restore: blob_backup_location = storage['blob_alt_location'] elif zip_restore: blob_backup_location = storage['blob_zip_location'] else: blob_backup_location = storage['blob_backup_location'] if not blob_backup_location: logger.error("No blob storage source specified") sys.exit(1) logger.info("Restoring blobs from %s to %s", blob_backup_location, blobdir) copyblobs.restore_blobs(blob_backup_location, blobdir, use_rsync=use_rsync, date=date, gzip_blob=gzip_blob, rsync_options=rsync_options) utils.execute_or_fail(post_command)
def restore_main( bin_dir, storages, verbose, backup_blobs, only_blobs, use_rsync, restore_snapshot=False, pre_command='', post_command='', archive_blob=False, alt_restore=False, rsync_options='', quick=True, zip_restore=False, blob_timestamps=False, incremental_blobs=False, **kwargs): """Main method, gets called by generated bin/restore.""" # First run several checks, and get the date that should be restored. date = restore_check( bin_dir, storages, verbose, backup_blobs, only_blobs, use_rsync, restore_snapshot=restore_snapshot, pre_command=pre_command, post_command=post_command, archive_blob=archive_blob, alt_restore=alt_restore, rsync_options=rsync_options, quick=quick, zip_restore=zip_restore, blob_timestamps=blob_timestamps, incremental_blobs=incremental_blobs, **kwargs) # Checks have passed, now do the real restore. if not only_blobs: result = repozorunner.restore_main( bin_dir, storages, verbose, date, restore_snapshot, alt_restore, zip_restore) if result: if backup_blobs: logger.error('Halting execution due to error; not restoring ' 'blobs.') else: logger.error('Halting execution due to error.') sys.exit(1) if not backup_blobs: utils.execute_or_fail(post_command) return for storage in storages: blobdir = storage['blobdir'] if not blobdir: continue blob_backup_location = storage['blob_backup_location'] logger.info('Restoring blobs from %s to %s', blob_backup_location, blobdir) result = copyblobs.restore_blobs( blob_backup_location, blobdir, use_rsync=use_rsync, date=date, archive_blob=archive_blob, rsync_options=rsync_options, timestamps=blob_timestamps, ) if result: logger.error('Halting execution due to error.') sys.exit(1) utils.execute_or_fail(post_command)
def restore_check( bin_dir, storages, verbose, backup_blobs, only_blobs, use_rsync, restore_snapshot=False, pre_command='', post_command='', archive_blob=False, alt_restore=False, rsync_options='', quick=True, zip_restore=False, blob_timestamps=False, **kwargs): """Method to check that a restore will work. Returns the chosen date, if any. """ explicit_restore_opts = [restore_snapshot, alt_restore, zip_restore] if sum([1 for opt in explicit_restore_opts if opt]) > 1: logger.error('Must use at most one option of restore_snapshot, ' 'alt_restore and zip_restore.') sys.exit(1) # Try to find a date in the command line arguments date = utils.get_date_from_args() if not kwargs.get('no_prompt'): question = '\n' if not only_blobs: question += 'This will replace the filestorage:\n' for storage in storages: question += ' {0}\n'.format(storage.get('datafs')) if backup_blobs: question += 'This will replace the blobstorage:\n' for storage in storages: if storage.get('blobdir'): question += ' {0}\n'.format(storage.get('blobdir')) question += 'Are you sure?' if not utils.ask(question, default=False, exact=True): logger.info('Not restoring.') sys.exit(0) utils.execute_or_fail(pre_command) # First run some checks. if not only_blobs: result = repozorunner.restore_main( bin_dir, storages, verbose, date, restore_snapshot, alt_restore, zip_restore, only_check=True) if result: logger.error('Halting execution: ' 'restoring filestorages would fail.') sys.exit(1) if backup_blobs: check_blobs( storages, use_rsync, restore_snapshot=restore_snapshot, archive_blob=archive_blob, alt_restore=alt_restore, rsync_options=rsync_options, zip_restore=zip_restore, blob_timestamps=blob_timestamps, date=date, ) return date
def backup_main( bin_dir, storages, keep, full, verbose, gzip, backup_blobs, only_blobs, use_rsync, keep_blob_days=0, pre_command='', post_command='', archive_blob=False, compress_blob=False, rsync_options='', quick=True, blob_timestamps=False, backup_method=config.STANDARD_BACKUP, incremental_blobs=False, **kwargs): """Main method, gets called by generated bin/backup.""" if backup_method not in config.BACKUP_METHODS: raise RuntimeError('Unknown backup method {0}.'.format(backup_method)) utils.execute_or_fail(pre_command) utils.check_folders( storages, backup_blobs=backup_blobs, only_blobs=only_blobs, backup_method=backup_method, ) if not only_blobs: result = repozorunner.backup_main( bin_dir, storages, keep, full, verbose, gzip, quick, backup_method, ) if result: if backup_blobs: logger.error( 'Halting execution due to error; not backing up blobs.') else: logger.error('Halting execution due to error.') sys.exit(1) if not backup_blobs: utils.execute_or_fail(post_command) return for storage in storages: blobdir = storage['blobdir'] if not blobdir: logger.info( 'No blob dir defined for %s storage', storage['storage']) continue blob_backup_location = None if backup_method == config.STANDARD_BACKUP: blob_backup_location = storage['blob_backup_location'] logger.info( 'Please wait while backing up blobs from %s to %s', blobdir, blob_backup_location) elif backup_method == config.SNAPSHOT_BACKUP: blob_backup_location = storage['blob_snapshot_location'] logger.info( 'Please wait while making snapshot of blobs from %s to %s', blobdir, blob_backup_location) elif backup_method == config.ZIP_BACKUP: blob_backup_location = storage['blob_zip_location'] logger.info( 'Please wait while backing up blobs from %s to %s', blobdir, blob_backup_location) if only_blobs: fs_backup_location = None else: fs_backup_location = storage['backup_location'] copyblobs.backup_blobs( blobdir, blob_backup_location, full, use_rsync, keep=keep, keep_blob_days=keep_blob_days, archive_blob=archive_blob, compress_blob=compress_blob, rsync_options=rsync_options, timestamps=blob_timestamps, fs_backup_location=fs_backup_location, incremental_blobs=incremental_blobs, ) utils.execute_or_fail(post_command)