def restore(self, keyspace, table, hosts, target_hosts):
table_log = "table: %s" % table if table else ''
snap_name = os.path.basename(self.snapshot.base_path)
logging.info("Restoring keyspace: %(keyspace)s %(table)s "
"from backup %(snap)s" %
dict(keyspace=keyspace, table=table_log, snap=snap_name))
check_lzop()
if self.local_restore:
logging.info("Backup files of %(host)s will be downloaded "
"in '%(dir)s'." %
dict(host=hosts[0], dir=self.restore_dir))
else:
logging.info("Backup files of the following host(s) will be "
"downloaded in '%(dir)s': %(hosts)s." %
dict(dir=self.restore_dir, hosts=', '.join(hosts)))
if self.run_sstableloader:
logging.info(
"After the downloading data will be streamed "
"to the following host(s) via sstableloader: %s." %
', '.join(target_hosts))
if not table:
table = ".*?"
bucket = self.s3connection.get_bucket(self.snapshot.s3_bucket,
validate=False)
# handle v2.0 and v2.1 table directory format
matcher_string = "(%(hosts)s).*/(%(keyspace)s)/(%(table)s-?[A-Za-z0-9]*)/" % dict(
hosts='|'.join(hosts), keyspace=keyspace, table=table)
self.keyspace_table_matcher = re.compile(matcher_string)
keys = []
tables = set()
for k in bucket.list(self.snapshot.base_path):
r = self.keyspace_table_matcher.search(k.name)
if not r:
continue
tables.add(r.group(3))
keys.append(k)
keyspace_path = os.path.join(self.restore_dir, keyspace)
self._delete_old_dir_and_create_new(keyspace_path, tables)
total_size = reduce(lambda s, k: s + k.size, keys, 0)
logging.info(
"Found %(files_count)d files, with total size of %(size)s." %
dict(files_count=len(keys), size=self._human_size(total_size)))
print("Found %(files_count)d files, with total size of %(size)s." %
dict(files_count=len(keys), size=self._human_size(total_size)))
self._download_keys(keys, total_size)
if target_hosts and self.run_sstableloader:
self._run_sstableloader(keyspace_path, tables, target_hosts,
self.cassandra_bin_dir)
logging.info("Restore completed.")
def main():
subparsers = base_parser.add_subparsers(
title='subcommands', dest='subcommand')
base_parser.add_argument(
'--incremental_backups', action='store_true', default=False)
put_parser = subparsers.add_parser(
'put', help="put files on s3 from a manifest")
manifest_parser = subparsers.add_parser(
'create-upload-manifest', help="put files on s3 from a manifest")
# put arguments
put_parser = add_s3_arguments(put_parser)
put_parser.add_argument(
'--bufsize',
required=False,
default=BUFFER_SIZE,
type=int,
help="Compress and upload buffer size")
put_parser.add_argument(
'--manifest',
required=True,
help="The manifest containing the files to put on s3")
put_parser.add_argument(
'--concurrency',
required=False,
default=DEFAULT_CONCURRENCY,
type=int,
help="Compress and upload concurrent processes")
put_parser.add_argument(
'--reduced-redundancy',
required=False,
default=DEFAULT_REDUCED_REDUNDANCY,
action="store_true",
help="Compress and upload concurrent processes")
put_parser.add_argument(
'--rate-limit',
required=False,
default=0,
type=int,
help="Limit the upload speed to S3 (by using 'pv'). Value expressed in kilobytes (*1024)")
# create-upload-manifest arguments
manifest_parser.add_argument('--snapshot_name', required=True, type=str)
manifest_parser.add_argument('--conf_path', required=True, type=str)
manifest_parser.add_argument('--manifest_path', required=True, type=str)
manifest_parser.add_argument(
'--snapshot_keyspaces', default='', required=False, type=str)
manifest_parser.add_argument(
'--snapshot_table', required=False, default='', type=str)
manifest_parser.add_argument(
'--exclude_tables', required=False, type=str)
args = base_parser.parse_args()
subcommand = args.subcommand
if subcommand == 'create-upload-manifest':
create_upload_manifest(
args.snapshot_name,
args.snapshot_keyspaces,
args.snapshot_table,
args.conf_path,
args.manifest_path,
args.exclude_tables,
args.incremental_backups
)
if subcommand == 'put':
check_lzop()
if args.rate_limit > 0:
check_pv()
put_from_manifest(
args.s3_bucket_name,
get_s3_connection_host(args.s3_bucket_region),
args.s3_ssenc,
args.s3_base_path,
args.aws_access_key_id,
args.aws_secret_access_key,
args.manifest,
args.bufsize,
args.reduced_redundancy,
args.rate_limit,
args.concurrency,
args.incremental_backups
)
args.manifest_path,
args.exclude_tables,
args.incremental_backups
)
if subcommand == 'put':
check_lzop()
if args.rate_limit > 0:
check_pv()
put_from_manifest(
args.s3_bucket_name,
get_s3_connection_host(args.s3_bucket_region),
args.s3_ssenc,
args.s3_base_path,
args.aws_access_key_id,
args.aws_secret_access_key,
args.manifest,
args.bufsize,
args.reduced_redundancy,
args.rate_limit,
args.concurrency,
args.incremental_backups
)
if __name__ == '__main__':
# TODO: if lzop is not available we should fail or run without it
check_lzop()
def main():
subparsers = base_parser.add_subparsers(
title='subcommands', dest='subcommand')
base_parser.add_argument(
'--incremental_backups', action='store_true', default=False)
put_parser = subparsers.add_parser(
'put', help="put files on s3 from a manifest")
manifest_parser = subparsers.add_parser(
'create-upload-manifest', help="put files on s3 from a manifest")
# put arguments
put_parser = add_s3_arguments(put_parser)
put_parser.add_argument(
'--bufsize',
required=False,
default=BUFFER_SIZE,
type=int,
help="Compress and upload buffer size")
put_parser.add_argument(
'--manifest',
required=True,
help="The manifest containing the files to put on s3")
put_parser.add_argument(
'--concurrency',
required=False,
default=DEFAULT_CONCURRENCY,
type=int,
help="Compress and upload concurrent processes")
# create-upload-manifest arguments
manifest_parser.add_argument('--snapshot_name', required=True, type=str)
manifest_parser.add_argument('--conf_path', required=True, type=str)
manifest_parser.add_argument('--manifest_path', required=True, type=str)
manifest_parser.add_argument(
'--snapshot_keyspaces', default='', required=False, type=str)
manifest_parser.add_argument(
'--snapshot_table', required=False, default='', type=str)
args = base_parser.parse_args()
subcommand = args.subcommand
if subcommand == 'create-upload-manifest':
create_upload_manifest(
args.snapshot_name,
args.snapshot_keyspaces,
args.snapshot_table,
args.conf_path,
args.manifest_path,
args.incremental_backups
)
if subcommand == 'put':
check_lzop()
put_from_manifest(
args.s3_bucket_name,
get_s3_connection_host(args.s3_bucket_region),
args.s3_ssenc,
args.s3_base_path,
args.aws_access_key_id,
args.aws_secret_access_key,
args.manifest,
args.bufsize,
args.concurrency,
args.incremental_backups
)
if subcommand == 'create-upload-manifest':
create_upload_manifest(
args.snapshot_name,
args.snapshot_keyspaces,
args.snapshot_table,
args.conf_path,
args.manifest_path,
args.incremental_backups
)
if subcommand == 'put':
check_lzop()
put_from_manifest(
args.s3_bucket_name,
get_s3_connection_host(args.s3_bucket_region),
args.s3_ssenc,
args.s3_base_path,
args.aws_access_key_id,
args.aws_secret_access_key,
args.manifest,
args.bufsize,
args.concurrency,
args.incremental_backups
)
if __name__ == '__main__':
# TODO: if lzop is not available we should fail or run without it
check_lzop()
def main():
subparsers = base_parser.add_subparsers(title="subcommands", dest="subcommand")
base_parser.add_argument("--incremental_backups", action="store_true", default=False)
put_parser = subparsers.add_parser("put", help="put files on s3 from a manifest")
manifest_parser = subparsers.add_parser("create-upload-manifest", help="put files on s3 from a manifest")
# put arguments
put_parser = add_s3_arguments(put_parser)
put_parser.add_argument(
"--bufsize", required=False, default=BUFFER_SIZE, type=int, help="Compress and upload buffer size"
)
put_parser.add_argument("--manifest", required=True, help="The manifest containing the files to put on s3")
put_parser.add_argument(
"--concurrency",
required=False,
default=DEFAULT_CONCURRENCY,
type=int,
help="Compress and upload concurrent processes",
)
put_parser.add_argument(
"--reduced-redundancy",
required=False,
default=DEFAULT_REDUCED_REDUNDANCY,
action="store_true",
help="Compress and upload concurrent processes",
)
put_parser.add_argument(
"--rate-limit",
required=False,
default=0,
type=int,
help="Limit the upload speed to S3 (by using 'pv'). Value expressed in kilobytes (*1024)",
)
# create-upload-manifest arguments
manifest_parser.add_argument("--snapshot_name", required=True, type=str)
manifest_parser.add_argument("--conf_path", required=True, type=str)
manifest_parser.add_argument("--manifest_path", required=True, type=str)
manifest_parser.add_argument("--snapshot_keyspaces", default="", required=False, type=str)
manifest_parser.add_argument("--snapshot_table", required=False, default="", type=str)
manifest_parser.add_argument("--exclude_tables", required=False, type=str)
args = base_parser.parse_args()
subcommand = args.subcommand
if subcommand == "create-upload-manifest":
create_upload_manifest(
args.snapshot_name,
args.snapshot_keyspaces,
args.snapshot_table,
args.conf_path,
args.manifest_path,
args.exclude_tables,
args.incremental_backups,
)
if subcommand == "put":
check_lzop()
if args.rate_limit > 0:
check_pv()
put_from_manifest(
args.s3_bucket_name,
get_s3_connection_host(args.s3_bucket_region),
args.s3_ssenc,
args.s3_base_path,
args.aws_access_key_id,
args.aws_secret_access_key,
args.manifest,
args.bufsize,
args.reduced_redundancy,
args.rate_limit,
args.concurrency,
args.incremental_backups,
)