def run_backup(args): _set_user_env_vars(args) env.hosts = args.hosts.split(',') env.keyspaces = args.keyspaces.split(',') if args.keyspaces else None if args.new_snapshot: create_snapshot = True else: existing_snapshot = SnapshotCollection( args.aws_access_key_id, args.aws_secret_access_key, args.s3_base_path, args.s3_bucket_name, get_s3_connection_host(args.s3_bucket_region) ).get_snapshot_for( hosts=env.hosts, keyspaces=env.keyspaces, table=args.table, name=Snapshot.make_snapshot_name() ) create_snapshot = existing_snapshot is None worker = BackupWorker( aws_access_key_id=args.aws_access_key_id, aws_secret_access_key=args.aws_secret_access_key, s3_bucket_region=args.s3_bucket_region, s3_ssenc=args.s3_ssenc, s3_connection_host=get_s3_connection_host(args.s3_bucket_region), cassandra_conf_path=args.cassandra_conf_path, cassandra_tools_bin_dir=args.cassandra_tools_bin_dir, cqlsh_user=args.cqlsh_user, cqlsh_password=args.cqlsh_password, backup_schema=args.backup_schema, buffer_size=args.buffer_size, use_sudo=args.use_sudo, use_local=args.use_local, connection_pool_size=args.connection_pool_size, exclude_tables=args.exclude_tables, reduced_redundancy=args.reduced_redundancy, rate_limit=args.rate_limit, quiet=args.quiet, nice=int(args.nice) ) if create_snapshot: logging.info("Make a new snapshot") snapshot = Snapshot( base_path=args.s3_base_path, s3_bucket=args.s3_bucket_name, hosts=env.hosts, keyspaces=env.keyspaces, table=args.table ) worker.snapshot(snapshot) else: logging.info("Add incrementals to snapshot {!s}".format( existing_snapshot)) worker.update_snapshot(existing_snapshot)
def restore_backup(args): _set_user_env_vars(args) env.host_string = args.host snapshots = SnapshotCollection( args.aws_access_key_id, args.aws_secret_access_key, args.s3_base_path, args.s3_bucket_name, get_s3_connection_host(args.s3_bucket_region)) if args.snapshot_name == 'LATEST': snapshot = snapshots.get_latest() else: snapshot = snapshots.get_snapshot_by_name(args.snapshot_name) worker = RestoreWorker( aws_access_key_id=args.aws_access_key_id, aws_secret_access_key=args.aws_secret_access_key, s3_bucket_region=args.s3_bucket_region, snapshot=snapshot, cassandra_tools_bin_dir=args.cassandra_tools_bin_dir, restore_dir=args.restore_dir, use_sudo=args.use_sudo, use_local=args.use_local) worker.restore(args.keyspace)
def main(): subparsers = base_parser.add_subparsers(title='subcommands', dest='subcommand') base_parser.add_argument('--incremental_backups', action='store_true', default=False) put_parser = subparsers.add_parser('put', help='put files on s3 from a manifest') manifest_parser = subparsers.add_parser( 'create-upload-manifest', help='put files on s3 from a manifest') # put arguments put_parser = add_s3_arguments(put_parser) put_parser.add_argument( '--manifest', required=True, help='The manifest containing the files to put on s3') put_parser.add_argument('--concurrency', required=False, default=DEFAULT_CONCURRENCY, type=int, help='Compress and upload concurrent processes') # create-upload-manifest arguments manifest_parser.add_argument('--snapshot_name', required=True, type=str) manifest_parser.add_argument('--snapshot_keyspaces', default='', required=False, type=str) manifest_parser.add_argument('--snapshot_table', required=False, default='', type=str) manifest_parser.add_argument('--data_path', required=True, type=str) manifest_parser.add_argument('--manifest_path', required=True, type=str) args = base_parser.parse_args() subcommand = args.subcommand if subcommand == 'create-upload-manifest': create_upload_manifest(args.snapshot_name, args.snapshot_keyspaces, args.snapshot_table, args.data_path, args.manifest_path, args.incremental_backups) if subcommand == 'put': check_lzop() put_from_manifest(args.s3_bucket_name, get_s3_connection_host(args.s3_bucket_region), args.s3_ssenc, args.s3_base_path, args.aws_access_key_id, args.aws_secret_access_key, args.manifest, args.concurrency, args.incremental_backups)
def run_backup(args): if args.user: env.user = args.user if args.password: env.password = args.password if args.sshport: env.port = args.sshport env.hosts = args.hosts.split(',') if args.new_snapshot: create_snapshot = True else: existing_snapshot = SnapshotCollection( args.aws_access_key_id, args.aws_secret_access_key, args.s3_base_path, args.s3_bucket_name ).get_snapshot_for( hosts=env.hosts, keyspaces=args.keyspaces, table=args.table ) create_snapshot = existing_snapshot is None worker = BackupWorker( aws_access_key_id=args.aws_access_key_id, aws_secret_access_key=args.aws_secret_access_key, s3_bucket_region=args.s3_bucket_region, s3_ssenc=args.s3_ssenc, s3_connection_host=get_s3_connection_host(args.s3_bucket_region), cassandra_data_path=args.cassandra_data_path, nodetool_path=args.nodetool_path, cassandra_bin_dir=args.cassandra_bin_dir, backup_schema=args.backup_schema, connection_pool_size=args.connection_pool_size ) if create_snapshot: logging.info('make a new snapshot') snapshot = Snapshot( base_path=args.s3_base_path, s3_bucket=args.s3_bucket_name, hosts=env.hosts, keyspaces=args.keyspaces, table=args.table ) worker.snapshot(snapshot) else: logging.info('add incrementals to snapshot %s' % existing_snapshot) worker.update_snapshot(existing_snapshot)
def main(): subparsers = base_parser.add_subparsers(title='subcommands', dest='subcommand') base_parser.add_argument('--incremental_backups', action='store_true', default=False) put_parser = subparsers.add_parser('put', help='put files on s3 from a manifest') manifest_parser = subparsers.add_parser('create-upload-manifest', help='put files on s3 from a manifest') # put arguments put_parser = add_s3_arguments(put_parser) put_parser.add_argument('--manifest', required=True, help='The manifest containing the files to put on s3') put_parser.add_argument('--concurrency', required=False, default=DEFAULT_CONCURRENCY, type=int, help='Compress and upload concurrent processes') # create-upload-manifest arguments manifest_parser.add_argument('--snapshot_name', required=True, type=str) manifest_parser.add_argument('--snapshot_keyspaces', default='', required=False, type=str) manifest_parser.add_argument('--snapshot_table', required=False, default='', type=str) manifest_parser.add_argument('--data_path', required=True, type=str) manifest_parser.add_argument('--manifest_path', required=True, type=str) args = base_parser.parse_args() subcommand = args.subcommand if subcommand == 'create-upload-manifest': create_upload_manifest( args.snapshot_name, args.snapshot_keyspaces, args.snapshot_table, args.data_path, args.manifest_path, args.incremental_backups ) if subcommand == 'put': check_lzop() put_from_manifest( args.s3_bucket_name, get_s3_connection_host(args.s3_bucket_region), args.s3_ssenc, args.s3_base_path, args.aws_access_key_id, args.aws_secret_access_key, args.manifest, args.concurrency, args.incremental_backups )
def run_backup(args): if args.user: env.user = args.user if args.password: env.password = args.password if args.sshkey - filename: env.key_filename = args.sshkey - filename if args.sshport: env.port = args.sshport env.hosts = args.hosts.split(',') if args.new_snapshot: create_snapshot = True else: existing_snapshot = SnapshotCollection( args.aws_access_key_id, args.aws_secret_access_key, args.s3_base_path, args.s3_bucket_name).get_snapshot_for(hosts=env.hosts, keyspaces=args.keyspaces, table=args.table) create_snapshot = existing_snapshot is None worker = BackupWorker(aws_access_key_id=args.aws_access_key_id, aws_secret_access_key=args.aws_secret_access_key, s3_bucket_region=args.s3_bucket_region, s3_ssenc=args.s3_ssenc, s3_connection_host=get_s3_connection_host( args.s3_bucket_region), cassandra_data_path=args.cassandra_data_path, nodetool_path=args.nodetool_path, cassandra_bin_dir=args.cassandra_bin_dir, backup_schema=args.backup_schema, connection_pool_size=args.connection_pool_size) if create_snapshot: logging.info('make a new snapshot') snapshot = Snapshot(base_path=args.s3_base_path, s3_bucket=args.s3_bucket_name, hosts=env.hosts, keyspaces=args.keyspaces, table=args.table) worker.snapshot(snapshot) else: logging.info('add incrementals to snapshot %s' % existing_snapshot) worker.update_snapshot(existing_snapshot)
def list_backups(args): snapshots = SnapshotCollection( args.aws_access_key_id, args.aws_secret_access_key, args.s3_base_path, args.s3_bucket_name, get_s3_connection_host(args.s3_bucket_region)) path_snapshots = defaultdict(list) for snapshot in snapshots: base_path = os.path.join(snapshot.base_path.split('/')[:-1]) path_snapshots[base_path].append(snapshot) for path, snapshots in path_snapshots.iteritems(): for snapshot in snapshots: print("\t {!r} hosts:{!r} keyspaces:{!r} table:{!r}".format( snapshot, snapshot.hosts, snapshot.keyspaces, snapshot.table)) print("------------------------{}".format('-' * len(path)))
def run_backup(args): if args.user: env.user = args.user if args.password: env.password = args.password if args.sshport: env.port = args.sshport if args.sshkey: env.key_filename = args.sshkey if not args.hosts: env.hosts = [socket.gethostname()] env.run = lambda cmd: local(cmd, capture=True) else: env.hosts = args.hosts.split(',') env.run = run keep_new_snapshot = args.keep_new_snapshot delete_old_snapshots = args.delete_old_snapshots delete_backups = args.delete_incremental_backups if args.new_snapshot: create_snapshot = True existing_snapshot = None else: if args.keep_new_snapshot: logging.warn('--new-snapshot not set. Ignoring --keep-new-snapshot ') keep_new_snapshot = False if args.delete_old_snapshots: logging.warn('--new-snapshot not set. Ignoring --delete-old-snapshots') delete_old_snapshots = False if args.delete_incremental_backups: logging.warn('--new-snapshot not set. Ignoring --delete-incremental-backup') delete_backups = False existing_snapshot = SnapshotCollection( args.aws_access_key_id, args.aws_secret_access_key, args.s3_base_path, args.s3_bucket_name ).get_snapshot_for( hosts=env.hosts, keyspaces=args.keyspaces, table=args.table ) create_snapshot = existing_snapshot is None worker = BackupWorker( aws_access_key_id=args.aws_access_key_id, aws_secret_access_key=args.aws_secret_access_key, s3_bucket_region=args.s3_bucket_region, s3_ssenc=args.s3_ssenc, s3_connection_host=get_s3_connection_host(args.s3_bucket_region), cassandra_data_path=args.cassandra_data_path, nodetool_path=args.nodetool_path, cassandra_bin_dir=args.cassandra_bin_dir, backup_schema=args.backup_schema, connection_pool_size=args.connection_pool_size, agent_path=args.agent_path, agent_virtualenv=args.agent_virtualenv, use_sudo=(not args.no_sudo) ) if create_snapshot: logging.info('make a new snapshot') snapshot = Snapshot( base_path=args.s3_base_path, s3_bucket=args.s3_bucket_name, hosts=env.hosts, keyspaces=args.keyspaces, table=args.table ) worker.snapshot(snapshot, keep_new_snapshot=keep_new_snapshot, delete_old_snapshots=delete_old_snapshots, delete_backups=delete_backups) else: logging.info('add incrementals to snapshot %s' % existing_snapshot) worker.update_snapshot(existing_snapshot)