def main():
parser = argparse.ArgumentParser(
description='Backup and restore for block devices.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-v',
'--verbose',
action='store_true',
help='verbose output')
parser.add_argument('-d',
'--debug',
action='store_true',
help='debug output')
parser.add_argument('-m',
'--machine-output',
action='store_true',
default=False)
parser.add_argument('-s',
'--skip-header',
action='store_true',
default=False)
parser.add_argument('-r',
'--human-readable',
action='store_true',
default=False)
parser.add_argument('-V',
'--version',
action='store_true',
help='Show version')
parser.add_argument('-c', '--configfile', default=None, type=str)
subparsers = parser.add_subparsers()
# INITDB
p = subparsers.add_parser(
'initdb',
help=
"Initialize the database by populating tables. This will not delete tables or data if they exist."
)
p.set_defaults(func='initdb')
# BACKUP
p = subparsers.add_parser('backup', help="Perform a backup.")
p.add_argument(
'source',
help=
'Source (url-like, e.g. file:///dev/sda or rbd://pool/imagename@snapshot)'
)
p.add_argument('name', help='Backup name (e.g. the hostname)')
p.add_argument('-s',
'--snapshot-name',
default='',
help='Snapshot name (e.g. the name of the rbd snapshot)')
p.add_argument('-r',
'--rbd',
default=None,
help='Hints as rbd json format')
p.add_argument('-f',
'--from-version',
default=None,
help='Use this version-uid as base')
p.add_argument('-c',
'--continue-version',
default=None,
help='Continue backup on this version-uid')
p.add_argument(
'-t',
'--tag',
default=None,
help=
'Use a specific tag (or multiple comma-separated tags) for the target backup version-uid'
)
p.add_argument(
'-e',
'--expire',
default='',
help='Expiration date (yyyy-mm-dd or "yyyy-mm-dd HH-MM-SS") (optional)'
)
p.set_defaults(func='backup')
# RESTORE
p = subparsers.add_parser('restore',
help="Restore a given backup to a given target.")
p.add_argument(
'-s',
'--sparse',
action='store_true',
help='Faster. Restore '
'only existing blocks (works only with file- and rbd-restore, not with lvm)'
)
p.add_argument('-f',
'--force',
action='store_true',
help='Force overwrite of existing files/devices/images')
p.add_argument(
'-c',
'--continue-from',
default=0,
help=
'Continue from this block (only use this for partially failed restores!)'
)
p.add_argument('version_uid')
p.add_argument(
'target',
help='Source (url-like, e.g. file:///dev/sda or rbd://pool/imagename)')
p.set_defaults(func='restore')
# PROTECT
p = subparsers.add_parser(
'protect',
help="Protect a backup version. Protected versions cannot be removed.")
p.add_argument('version_uid')
p.set_defaults(func='protect')
# UNPROTECT
p = subparsers.add_parser(
'unprotect',
help="Unprotect a backup version. Unprotected versions can be removed."
)
p.add_argument('version_uid')
p.set_defaults(func='unprotect')
# RM
p = subparsers.add_parser(
'rm',
help=
"Remove a given backup version. This will only remove meta data and you will have to cleanup after this."
)
p.add_argument(
'-f',
'--force',
action='store_true',
help=
"Force removal of version, even if it's younger than the configured disallow_rm_when_younger_than_days."
)
p.add_argument('version_uid')
p.set_defaults(func='rm')
# SCRUB
p = subparsers.add_parser(
'scrub', help="Scrub a given backup and check for consistency.")
p.add_argument(
'-s',
'--source',
default=None,
help=
"Source, optional. If given, check if source matches backup in addition to checksum tests. url-like format as in backup."
)
p.add_argument(
'-p',
'--percentile',
default=100,
help=
"Only check PERCENTILE percent of the blocks (value 0..100). Default: 100"
)
p.add_argument('version_uid')
p.set_defaults(func='scrub')
# Export
p = subparsers.add_parser(
'export', help="Export the metadata of a backup uid into a file.")
p.add_argument('version_uid')
p.add_argument('filename',
help="Export into this filename ('-' is for stdout)")
p.set_defaults(func='export')
# Import
p = subparsers.add_parser(
'import', help="Import the metadata of a backup from a file.")
p.add_argument('filename', help="Read from this file ('-' is for stdin)")
p.set_defaults(func='import_')
# CLEANUP
p = subparsers.add_parser('cleanup', help="Clean unreferenced blobs.")
p.add_argument(
'-f',
'--full',
action='store_true',
default=False,
help=
'Do a full cleanup. This will read the full metadata from the data backend (i.e. backup storage) '
'and compare it to the metadata in the meta backend. Unused data will then be deleted. '
'This is a slow, but complete process. A full cleanup must not be run parallel to ANY other backy '
'jobs.')
p.add_argument(
'-p',
'--prefix',
default=None,
help=
'If you perform a full cleanup, you may add --prefix to only cleanup block uids starting '
'with this prefix. This is for iterative cleanups. Example: '
'cleanup --full --prefix=a')
p.add_argument(
'--dangerous-force',
action='store_true',
default=False,
help='Seriously, do not use this outside of testing and development.')
p.set_defaults(func='cleanup')
# LS
p = subparsers.add_parser('ls', help="List existing backups.")
p.add_argument('name',
nargs='?',
default=None,
help='Show versions for this name only')
p.add_argument('-s',
'--snapshot-name',
default=None,
help="Limit output to this snapshot name")
p.add_argument('-t',
'--tag',
default=None,
help="Limit output to this tag")
p.add_argument('-e',
'--expired',
action='store_true',
default=False,
help="Only list expired versions (expired < now)")
p.add_argument(
'-f',
'--fields',
default=
"date,name,snapshot_name,size,size_bytes,uid,valid,protected,tags,expire",
help=
"Show these fields (comma separated). Available: date,name,snapshot_name,size,size_bytes,uid,valid,protected,tags,expire"
)
p.set_defaults(func='ls')
# STATS
p = subparsers.add_parser('stats', help="Show statistics")
p.add_argument('version_uid',
nargs='?',
default=None,
help='Show statistics for this version')
p.add_argument(
'-f',
'--fields',
default=
"date,uid,name,size bytes,size blocks,bytes read,blocks read,bytes written,blocks written,bytes dedup,blocks dedup,bytes sparse,blocks sparse,duration (s)",
help=
"Show these fields (comma separated). Available: date,uid,name,size bytes,size blocks,bytes read,blocks read,bytes written,blocks written,bytes dedup,blocks dedup,bytes sparse,blocks sparse,duration (s)"
)
p.add_argument('-l',
'--limit',
default=None,
help="Limit output to this number (default: unlimited)")
p.set_defaults(func='stats')
# diff-meta
p = subparsers.add_parser('diff-meta',
help="Output a diff between two versions")
p.add_argument('version_uid1', help='Left version')
p.add_argument('version_uid2', help='Right version')
p.set_defaults(func='diff_meta')
# disk usage
p = subparsers.add_parser(
'du', help="Get disk usage for a version or for all versions")
p.add_argument('version_uid',
nargs='?',
default=None,
help='Show disk usage for this version')
p.add_argument(
'-f',
'--fields',
default=
"Real,Null,Dedup Own,Dedup Others,Individual,Est. Space,Est. Space freed",
help=
"Show these fields (comma separated). Available: Real,Null,Dedup Own,Dedup Others,Individual,Est. Space,Est. Space freed)"
)
p.set_defaults(func='du')
# FUSE
p = subparsers.add_parser('fuse', help="Fuse mount backy backups")
p.add_argument('mount', help='Mountpoint')
p.set_defaults(func='fuse')
# Re-Keying
p = subparsers.add_parser(
'rekey',
help=
"Re-Key all blocks in backy2 with a new key in the config. This will NOT encrypt unencrypted blocks or recrypt existing blocks."
)
p.add_argument('oldkey', help='The old key as it was found in the config')
p.set_defaults(func='rekey')
# Migrate encryption
p = subparsers.add_parser(
'migrate-encryption',
help=
"Create a new version with blocks migrated/encrypted to the latest encryption version."
)
p.add_argument('version_uid', help='The version uid to migrate')
p.set_defaults(func='migrate_encryption')
# ADD TAG
p = subparsers.add_parser(
'add-tag',
help=
"Add a named tag (or many comma-separated tags) to a backup version.")
p.add_argument('version_uid')
p.add_argument('name')
p.set_defaults(func='add_tag')
# REMOVE TAG
p = subparsers.add_parser(
'remove-tag',
help=
"Remove a named tag (or many comma-separated tags) from a backup version."
)
p.add_argument('version_uid')
p.add_argument('name')
p.set_defaults(func='remove_tag')
# EXPIRE
p = subparsers.add_parser(
'expire',
help=
"""Set expiration date for a backup version. Date format is yyyy-mm-dd or "yyyy-mm-dd HH:MM:SS" (e.g. 2020-01-23). HINT: Create with 'date +"%%Y-%%m-%%d" -d "today + 7 days"'"""
)
p.add_argument('version_uid')
p.add_argument('expire')
p.set_defaults(func='expire')
# DUE
p = subparsers.add_parser(
'due',
help=
"""Based on the schedulers in the config file, calculate the due backups including tags."""
)
p.add_argument(
'name',
nargs='?',
default=None,
help=
'Show due backups for this version name (optional, if not given, show due backups for all names).'
)
p.add_argument(
'-s',
'--schedulers',
default="daily,weekly,monthly",
help=
"Use these schedulers as defined in backy.cfg (default: daily,weekly,monthly)"
)
p.add_argument(
'-f',
'--fields',
default="name,schedulers,expire_date,due_since",
help=
"Show these fields (comma separated). Available: name,schedulers,expire_date"
)
p.set_defaults(func='due')
# SLA
p = subparsers.add_parser(
'sla',
help=
"""Based on the schedulers in the config file, calculate the information about SLA."""
)
p.add_argument(
'name',
nargs='?',
default=None,
help=
'Show SLA breaches for this version name (optional, if not given, show SLA breaches for all names).'
)
p.add_argument(
'-s',
'--schedulers',
default="daily,weekly,monthly",
help=
"Use these schedulers as defined in backy.cfg (default: daily,weekly,monthly)"
)
p.add_argument(
'-f',
'--fields',
default="name,breach",
help="Show these fields (comma separated). Available: name,breach")
p.set_defaults(func='sla')
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if not hasattr(args, 'func'):
parser.print_usage()
sys.exit(1)
if args.verbose:
console_level = logging.DEBUG
#elif args.func == 'scheduler':
#console_level = logging.INFO
else:
console_level = logging.INFO
if args.debug:
debug = True
else:
debug = False
if args.configfile is not None and args.configfile != '':
try:
cfg = open(args.configfile, 'r', encoding='utf-8').read()
except FileNotFoundError:
logger.error('File not found: {}'.format(args.configfile))
sys.exit(1)
Config = partial(_Config, cfg=cfg)
else:
Config = partial(_Config, conf_name='backy')
config = Config(section='DEFAULTS')
# logging ERROR only when machine output is selected
if args.machine_output:
init_logging(config.get('logfile'), logging.ERROR, debug)
else:
init_logging(config.get('logfile'), console_level, debug)
commands = Commands(args.machine_output, args.skip_header,
args.human_readable, Config)
func = getattr(commands, args.func)
# Pass over to function
func_args = dict(args._get_kwargs())
del func_args['configfile']
del func_args['func']
del func_args['verbose']
del func_args['debug']
del func_args['version']
del func_args['machine_output']
del func_args['skip_header']
del func_args['human_readable']
try:
logger.debug('backup.{0}(**{1!r})'.format(args.func, func_args))
func(**func_args)
logger.info('Backy complete.\n')
sys.exit(0)
except Exception as e:
if args.debug:
logger.error('Unexpected exception')
logger.exception(e)
else:
logger.error(e)
logger.error('Backy failed.\n')
sys.exit(100)
class TestPath():
path = '_smoketest'
def __enter__(self):
os.mkdir(self.path)
return self.path
def __exit__(self, type, value, traceback):
shutil.rmtree(self.path)
with TestPath() as testpath:
from_version = None
init_logging(testpath + '/backy.log', logging.INFO)
version_uids = []
old_size = 0
initdb = True
for i in range(100):
print('Run {}'.format(i + 1))
hints = []
if old_size and random.randint(
0, 10) == 0: # every 10th time or so do not apply any changes.
size = old_size
else:
size = 32 * 4 * kB + random.randint(-4 * kB, 4 * kB)
if size > old_size:
hints.append({
'offset': old_size,
def main():
parser = argparse.ArgumentParser(
description='Backup and restore for block devices.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'-v', '--verbose', action='store_true', help='verbose output')
parser.add_argument(
'-m', '--machine-output', action='store_true', default=False)
parser.add_argument(
'-V', '--version', action='store_true', help='Show version')
subparsers = parser.add_subparsers()
# BACKUP
p = subparsers.add_parser(
'backup',
help="Perform a backup.")
p.add_argument(
'source',
help='Source file')
p.add_argument(
'name',
help='Backup name')
p.add_argument('-r', '--rbd', default=None, help='Hints as rbd json format')
p.add_argument('-f', '--from-version', default=None, help='Use this version-uid as base')
p.set_defaults(func='backup')
# RESTORE
p = subparsers.add_parser(
'restore',
help="Restore a given backup with level to a given target.")
p.add_argument('-s', '--sparse', action='store_true', help='Write restore file sparse (does not work with legacy devices)')
p.add_argument('version_uid')
p.add_argument('target')
p.set_defaults(func='restore')
# RM
p = subparsers.add_parser(
'rm',
help="Remove a given backup version. This will only remove meta data and you will have to cleanup after this.")
p.add_argument('version_uid')
p.set_defaults(func='rm')
# SCRUB
p = subparsers.add_parser(
'scrub',
help="Scrub a given backup and check for consistency.")
p.add_argument('-s', '--source', default=None,
help="Source, optional. If given, check if source matches backup in addition to checksum tests.")
p.add_argument('-p', '--percentile', default=100,
help="Only check PERCENTILE percent of the blocks (value 0..100). Default: 100")
p.add_argument('version_uid')
p.set_defaults(func='scrub')
# Export
p = subparsers.add_parser(
'export',
help="Export the metadata of a backup uid into a file.")
p.add_argument('version_uid')
p.add_argument('filename', help="Export into this filename ('-' is for stdout)")
p.set_defaults(func='export')
# Import
p = subparsers.add_parser(
'import',
help="Import the metadata of a backup from a file.")
p.add_argument('filename', help="Read from this file ('-' is for stdin)")
p.set_defaults(func='import_')
# CLEANUP
p = subparsers.add_parser(
'cleanup',
help="Clean unreferenced blobs.")
p.add_argument(
'-f', '--full', action='store_true', default=False,
help='Do a full cleanup. This will read the full metadata from the data backend (i.e. backup storage) '
'and compare it to the metadata in the meta backend. Unused data will then be deleted. '
'This is a slow, but complete process. A full cleanup must not be run parallel to ANY other backy '
'jobs.')
p.add_argument(
'-p', '--prefix', default=None,
help='If you perform a full cleanup, you may add --prefix to only cleanup block uids starting '
'with this prefix. This is for iterative cleanups. Example: '
'cleanup --full --prefix=a')
p.set_defaults(func='cleanup')
# LS
p = subparsers.add_parser(
'ls',
help="List existing backups.")
p.add_argument('version_uid', nargs='?', default=None, help='Show verbose blocks for this version')
p.set_defaults(func='ls')
# STATS
p = subparsers.add_parser(
'stats',
help="Show statistics")
p.add_argument('version_uid', nargs='?', default=None, help='Show statistics for this version')
p.add_argument('-l', '--limit', default=None,
help="Limit output to this number (default: unlimited)")
p.set_defaults(func='stats')
# diff-meta
p = subparsers.add_parser(
'diff-meta',
help="Output a diff between two versions")
p.add_argument('version_uid1', help='Left version')
p.add_argument('version_uid2', help='Right version')
p.set_defaults(func='diff_meta')
# NBD
p = subparsers.add_parser(
'nbd',
help="Start an nbd server")
p.add_argument('version_uid', nargs='?', default=None, help='Start an nbd server for this version')
p.add_argument('-a', '--bind-address', default='127.0.0.1',
help="Bind to this ip address (default: 127.0.0.1)")
p.add_argument('-p', '--bind-port', default=10809,
help="Bind to this port (default: 10809)")
p.add_argument(
'-r', '--read-only', action='store_true', default=False,
help='Read only if set, otherwise a copy on write backup is created.')
p.set_defaults(func='nbd')
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if not hasattr(args, 'func'):
parser.print_usage()
sys.exit(1)
if args.verbose:
console_level = logging.DEBUG
#elif args.func == 'scheduler':
#console_level = logging.INFO
else:
console_level = logging.INFO
Config = partial(_Config, conf_name='backy')
config = Config(section='DEFAULTS')
init_logging(config.get('logfile'), console_level)
commands = Commands(args.machine_output, Config)
func = getattr(commands, args.func)
# Pass over to function
func_args = dict(args._get_kwargs())
del func_args['func']
del func_args['verbose']
del func_args['version']
del func_args['machine_output']
try:
logger.debug('backup.{0}(**{1!r})'.format(args.func, func_args))
func(**func_args)
logger.info('Backy complete.\n')
sys.exit(0)
except Exception as e:
logger.error('Unexpected exception')
logger.exception(e)
logger.info('Backy failed.\n')
sys.exit(100)
class TestPath():
path = '_smoketest'
def __enter__(self):
os.mkdir(self.path)
return self.path
def __exit__(self, type, value, traceback):
shutil.rmtree(self.path)
with TestPath() as testpath:
from_version = None
init_logging(testpath+'/backy.log', logging.INFO)
version_uids = []
for i in range(100):
size = 32*4*kB + random.randint(-4*kB, 4*kB)
print('Run {}'.format(i+1))
hints = []
for j in range(random.randint(0, 10)): # up to 10 changes
if random.randint(0, 1):
patch_size = random.randint(0, 4*kB)
data = os.urandom(patch_size)
#exists = True
exists = "true"
else:
patch_size = random.randint(0, 4*4*kB) # we want full blocks sometimes
data = b'\0' * patch_size
def main():
parser = argparse.ArgumentParser(
description='Backup and restore for block devices.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'-v', '--verbose', action='store_true', help='verbose output')
parser.add_argument(
'-m', '--machine-output', action='store_true', default=False)
parser.add_argument(
'-V', '--version', action='store_true', help='Show version')
subparsers = parser.add_subparsers()
# BACKUP
p = subparsers.add_parser(
'backup',
help="Perform a backup.")
p.add_argument(
'source',
help='Source (url-like, e.g. file:///dev/sda or rbd://pool/imagename@snapshot)')
p.add_argument(
'name',
help='Backup name')
p.add_argument('-r', '--rbd', default=None, help='Hints as rbd json format')
p.add_argument('-f', '--from-version', default=None, help='Use this version-uid as base')
p.set_defaults(func='backup')
# RESTORE
p = subparsers.add_parser(
'restore',
help="Restore a given backup to a given target.")
p.add_argument('-s', '--sparse', action='store_true', help='Faster. Restore '
'only existing blocks (works only with file- and rbd-restore, not with lvm)')
p.add_argument('-f', '--force', action='store_true', help='Force overwrite of existing files/devices/images')
p.add_argument('version_uid')
p.add_argument('target',
help='Source (url-like, e.g. file:///dev/sda or rbd://pool/imagename)')
p.set_defaults(func='restore')
# PROTECT
p = subparsers.add_parser(
'protect',
help="Protect a backup version. Protected versions cannot be removed.")
p.add_argument('version_uid')
p.set_defaults(func='protect')
# UNPROTECT
p = subparsers.add_parser(
'unprotect',
help="Unprotect a backup version. Unprotected versions can be removed.")
p.add_argument('version_uid')
p.set_defaults(func='unprotect')
# RM
p = subparsers.add_parser(
'rm',
help="Remove a given backup version. This will only remove meta data and you will have to cleanup after this.")
p.add_argument('-f', '--force', action='store_true', help="Force removal of version, even if it's younger than the configured disallow_rm_when_younger_than_days.")
p.add_argument('version_uid')
p.set_defaults(func='rm')
# SCRUB
p = subparsers.add_parser(
'scrub',
help="Scrub a given backup and check for consistency.")
p.add_argument('-s', '--source', default=None,
help="Source, optional. If given, check if source matches backup in addition to checksum tests. url-like format as in backup.")
p.add_argument('-p', '--percentile', default=100,
help="Only check PERCENTILE percent of the blocks (value 0..100). Default: 100")
p.add_argument('version_uid')
p.set_defaults(func='scrub')
# Export
p = subparsers.add_parser(
'export',
help="Export the metadata of a backup uid into a file.")
p.add_argument('version_uid')
p.add_argument('filename', help="Export into this filename ('-' is for stdout)")
p.set_defaults(func='export')
# Import
p = subparsers.add_parser(
'import',
help="Import the metadata of a backup from a file.")
p.add_argument('filename', help="Read from this file ('-' is for stdin)")
p.set_defaults(func='import_')
# CLEANUP
p = subparsers.add_parser(
'cleanup',
help="Clean unreferenced blobs.")
p.add_argument(
'-f', '--full', action='store_true', default=False,
help='Do a full cleanup. This will read the full metadata from the data backend (i.e. backup storage) '
'and compare it to the metadata in the meta backend. Unused data will then be deleted. '
'This is a slow, but complete process. A full cleanup must not be run parallel to ANY other backy '
'jobs.')
p.add_argument(
'-p', '--prefix', default=None,
help='If you perform a full cleanup, you may add --prefix to only cleanup block uids starting '
'with this prefix. This is for iterative cleanups. Example: '
'cleanup --full --prefix=a')
p.set_defaults(func='cleanup')
# LS
p = subparsers.add_parser(
'ls',
help="List existing backups.")
p.add_argument('version_uid', nargs='?', default=None, help='Show verbose blocks for this version')
p.set_defaults(func='ls')
# STATS
p = subparsers.add_parser(
'stats',
help="Show statistics")
p.add_argument('version_uid', nargs='?', default=None, help='Show statistics for this version')
p.add_argument('-l', '--limit', default=None,
help="Limit output to this number (default: unlimited)")
p.set_defaults(func='stats')
# diff-meta
p = subparsers.add_parser(
'diff-meta',
help="Output a diff between two versions")
p.add_argument('version_uid1', help='Left version')
p.add_argument('version_uid2', help='Right version')
p.set_defaults(func='diff_meta')
# NBD
p = subparsers.add_parser(
'nbd',
help="Start an nbd server")
p.add_argument('version_uid', nargs='?', default=None, help='Start an nbd server for this version')
p.add_argument('-a', '--bind-address', default='127.0.0.1',
help="Bind to this ip address (default: 127.0.0.1)")
p.add_argument('-p', '--bind-port', default=10809,
help="Bind to this port (default: 10809)")
p.add_argument(
'-r', '--read-only', action='store_true', default=False,
help='Read only if set, otherwise a copy on write backup is created.')
p.set_defaults(func='nbd')
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if not hasattr(args, 'func'):
parser.print_usage()
sys.exit(1)
if args.verbose:
console_level = logging.DEBUG
#elif args.func == 'scheduler':
#console_level = logging.INFO
else:
console_level = logging.INFO
Config = partial(_Config, conf_name='backy')
config = Config(section='DEFAULTS')
init_logging(config.get('logfile'), console_level)
commands = Commands(args.machine_output, Config)
func = getattr(commands, args.func)
# Pass over to function
func_args = dict(args._get_kwargs())
del func_args['func']
del func_args['verbose']
del func_args['version']
del func_args['machine_output']
try:
logger.debug('backup.{0}(**{1!r})'.format(args.func, func_args))
func(**func_args)
logger.info('Backy complete.\n')
sys.exit(0)
except Exception as e:
logger.error('Unexpected exception')
logger.exception(e)
logger.info('Backy failed.\n')
sys.exit(100)