def restore(self, keyspace, table, hosts, target_hosts): table_log = "table: %s" % table if table else '' snap_name = os.path.basename(self.snapshot.base_path) logging.info("Restoring keyspace: %(keyspace)s %(table)s " "from backup %(snap)s" % dict(keyspace=keyspace, table=table_log, snap=snap_name)) check_lzop() if self.local_restore: logging.info("Backup files of %(host)s will be downloaded " "in '%(dir)s'." % dict(host=hosts[0], dir=self.restore_dir)) else: logging.info("Backup files of the following host(s) will be " "downloaded in '%(dir)s': %(hosts)s." % dict(dir=self.restore_dir, hosts=', '.join(hosts))) if self.run_sstableloader: logging.info( "After the downloading data will be streamed " "to the following host(s) via sstableloader: %s." % ', '.join(target_hosts)) if not table: table = ".*?" bucket = self.s3connection.get_bucket(self.snapshot.s3_bucket, validate=False) # handle v2.0 and v2.1 table directory format matcher_string = "(%(hosts)s).*/(%(keyspace)s)/(%(table)s-?[A-Za-z0-9]*)/" % dict( hosts='|'.join(hosts), keyspace=keyspace, table=table) self.keyspace_table_matcher = re.compile(matcher_string) keys = [] tables = set() for k in bucket.list(self.snapshot.base_path): r = self.keyspace_table_matcher.search(k.name) if not r: continue tables.add(r.group(3)) keys.append(k) keyspace_path = os.path.join(self.restore_dir, keyspace) self._delete_old_dir_and_create_new(keyspace_path, tables) total_size = reduce(lambda s, k: s + k.size, keys, 0) logging.info( "Found %(files_count)d files, with total size of %(size)s." % dict(files_count=len(keys), size=self._human_size(total_size))) print("Found %(files_count)d files, with total size of %(size)s." % dict(files_count=len(keys), size=self._human_size(total_size))) self._download_keys(keys, total_size) if target_hosts and self.run_sstableloader: self._run_sstableloader(keyspace_path, tables, target_hosts, self.cassandra_bin_dir) logging.info("Restore completed.")
def main(): subparsers = base_parser.add_subparsers( title='subcommands', dest='subcommand') base_parser.add_argument( '--incremental_backups', action='store_true', default=False) put_parser = subparsers.add_parser( 'put', help="put files on s3 from a manifest") manifest_parser = subparsers.add_parser( 'create-upload-manifest', help="put files on s3 from a manifest") # put arguments put_parser = add_s3_arguments(put_parser) put_parser.add_argument( '--bufsize', required=False, default=BUFFER_SIZE, type=int, help="Compress and upload buffer size") put_parser.add_argument( '--manifest', required=True, help="The manifest containing the files to put on s3") put_parser.add_argument( '--concurrency', required=False, default=DEFAULT_CONCURRENCY, type=int, help="Compress and upload concurrent processes") put_parser.add_argument( '--reduced-redundancy', required=False, default=DEFAULT_REDUCED_REDUNDANCY, action="store_true", help="Compress and upload concurrent processes") put_parser.add_argument( '--rate-limit', required=False, default=0, type=int, help="Limit the upload speed to S3 (by using 'pv'). Value expressed in kilobytes (*1024)") # create-upload-manifest arguments manifest_parser.add_argument('--snapshot_name', required=True, type=str) manifest_parser.add_argument('--conf_path', required=True, type=str) manifest_parser.add_argument('--manifest_path', required=True, type=str) manifest_parser.add_argument( '--snapshot_keyspaces', default='', required=False, type=str) manifest_parser.add_argument( '--snapshot_table', required=False, default='', type=str) manifest_parser.add_argument( '--exclude_tables', required=False, type=str) args = base_parser.parse_args() subcommand = args.subcommand if subcommand == 'create-upload-manifest': create_upload_manifest( args.snapshot_name, args.snapshot_keyspaces, args.snapshot_table, args.conf_path, args.manifest_path, args.exclude_tables, args.incremental_backups ) if subcommand == 'put': check_lzop() if args.rate_limit > 0: check_pv() put_from_manifest( args.s3_bucket_name, get_s3_connection_host(args.s3_bucket_region), args.s3_ssenc, args.s3_base_path, args.aws_access_key_id, args.aws_secret_access_key, args.manifest, args.bufsize, args.reduced_redundancy, args.rate_limit, args.concurrency, args.incremental_backups )
args.manifest_path, args.exclude_tables, args.incremental_backups ) if subcommand == 'put': check_lzop() if args.rate_limit > 0: check_pv() put_from_manifest( args.s3_bucket_name, get_s3_connection_host(args.s3_bucket_region), args.s3_ssenc, args.s3_base_path, args.aws_access_key_id, args.aws_secret_access_key, args.manifest, args.bufsize, args.reduced_redundancy, args.rate_limit, args.concurrency, args.incremental_backups ) if __name__ == '__main__': # TODO: if lzop is not available we should fail or run without it check_lzop()
def main(): subparsers = base_parser.add_subparsers( title='subcommands', dest='subcommand') base_parser.add_argument( '--incremental_backups', action='store_true', default=False) put_parser = subparsers.add_parser( 'put', help="put files on s3 from a manifest") manifest_parser = subparsers.add_parser( 'create-upload-manifest', help="put files on s3 from a manifest") # put arguments put_parser = add_s3_arguments(put_parser) put_parser.add_argument( '--bufsize', required=False, default=BUFFER_SIZE, type=int, help="Compress and upload buffer size") put_parser.add_argument( '--manifest', required=True, help="The manifest containing the files to put on s3") put_parser.add_argument( '--concurrency', required=False, default=DEFAULT_CONCURRENCY, type=int, help="Compress and upload concurrent processes") # create-upload-manifest arguments manifest_parser.add_argument('--snapshot_name', required=True, type=str) manifest_parser.add_argument('--conf_path', required=True, type=str) manifest_parser.add_argument('--manifest_path', required=True, type=str) manifest_parser.add_argument( '--snapshot_keyspaces', default='', required=False, type=str) manifest_parser.add_argument( '--snapshot_table', required=False, default='', type=str) args = base_parser.parse_args() subcommand = args.subcommand if subcommand == 'create-upload-manifest': create_upload_manifest( args.snapshot_name, args.snapshot_keyspaces, args.snapshot_table, args.conf_path, args.manifest_path, args.incremental_backups ) if subcommand == 'put': check_lzop() put_from_manifest( args.s3_bucket_name, get_s3_connection_host(args.s3_bucket_region), args.s3_ssenc, args.s3_base_path, args.aws_access_key_id, args.aws_secret_access_key, args.manifest, args.bufsize, args.concurrency, args.incremental_backups )
if subcommand == 'create-upload-manifest': create_upload_manifest( args.snapshot_name, args.snapshot_keyspaces, args.snapshot_table, args.conf_path, args.manifest_path, args.incremental_backups ) if subcommand == 'put': check_lzop() put_from_manifest( args.s3_bucket_name, get_s3_connection_host(args.s3_bucket_region), args.s3_ssenc, args.s3_base_path, args.aws_access_key_id, args.aws_secret_access_key, args.manifest, args.bufsize, args.concurrency, args.incremental_backups ) if __name__ == '__main__': # TODO: if lzop is not available we should fail or run without it check_lzop()
def main(): subparsers = base_parser.add_subparsers(title="subcommands", dest="subcommand") base_parser.add_argument("--incremental_backups", action="store_true", default=False) put_parser = subparsers.add_parser("put", help="put files on s3 from a manifest") manifest_parser = subparsers.add_parser("create-upload-manifest", help="put files on s3 from a manifest") # put arguments put_parser = add_s3_arguments(put_parser) put_parser.add_argument( "--bufsize", required=False, default=BUFFER_SIZE, type=int, help="Compress and upload buffer size" ) put_parser.add_argument("--manifest", required=True, help="The manifest containing the files to put on s3") put_parser.add_argument( "--concurrency", required=False, default=DEFAULT_CONCURRENCY, type=int, help="Compress and upload concurrent processes", ) put_parser.add_argument( "--reduced-redundancy", required=False, default=DEFAULT_REDUCED_REDUNDANCY, action="store_true", help="Compress and upload concurrent processes", ) put_parser.add_argument( "--rate-limit", required=False, default=0, type=int, help="Limit the upload speed to S3 (by using 'pv'). Value expressed in kilobytes (*1024)", ) # create-upload-manifest arguments manifest_parser.add_argument("--snapshot_name", required=True, type=str) manifest_parser.add_argument("--conf_path", required=True, type=str) manifest_parser.add_argument("--manifest_path", required=True, type=str) manifest_parser.add_argument("--snapshot_keyspaces", default="", required=False, type=str) manifest_parser.add_argument("--snapshot_table", required=False, default="", type=str) manifest_parser.add_argument("--exclude_tables", required=False, type=str) args = base_parser.parse_args() subcommand = args.subcommand if subcommand == "create-upload-manifest": create_upload_manifest( args.snapshot_name, args.snapshot_keyspaces, args.snapshot_table, args.conf_path, args.manifest_path, args.exclude_tables, args.incremental_backups, ) if subcommand == "put": check_lzop() if args.rate_limit > 0: check_pv() put_from_manifest( args.s3_bucket_name, get_s3_connection_host(args.s3_bucket_region), args.s3_ssenc, args.s3_base_path, args.aws_access_key_id, args.aws_secret_access_key, args.manifest, args.bufsize, args.reduced_redundancy, args.rate_limit, args.concurrency, args.incremental_backups, )