Ejemplo n.º 1
0
    def restore(self, keyspace, table, hosts, target_hosts):
        table_log = "table: %s" % table if table else ''
        snap_name = os.path.basename(self.snapshot.base_path)
        logging.info("Restoring keyspace: %(keyspace)s %(table)s "
                     "from backup %(snap)s" %
                     dict(keyspace=keyspace, table=table_log, snap=snap_name))
        check_lzop()

        if self.local_restore:
            logging.info("Backup files of %(host)s will be downloaded "
                         "in '%(dir)s'." %
                         dict(host=hosts[0], dir=self.restore_dir))
        else:
            logging.info("Backup files of the following host(s) will be "
                         "downloaded in '%(dir)s': %(hosts)s." %
                         dict(dir=self.restore_dir, hosts=', '.join(hosts)))

            if self.run_sstableloader:
                logging.info(
                    "After the downloading data will be streamed "
                    "to the following host(s) via sstableloader: %s." %
                    ', '.join(target_hosts))
        if not table:
            table = ".*?"

        bucket = self.s3connection.get_bucket(self.snapshot.s3_bucket,
                                              validate=False)

        # handle v2.0 and v2.1 table directory format
        matcher_string = "(%(hosts)s).*/(%(keyspace)s)/(%(table)s-?[A-Za-z0-9]*)/" % dict(
            hosts='|'.join(hosts), keyspace=keyspace, table=table)
        self.keyspace_table_matcher = re.compile(matcher_string)

        keys = []
        tables = set()

        for k in bucket.list(self.snapshot.base_path):
            r = self.keyspace_table_matcher.search(k.name)
            if not r:
                continue

            tables.add(r.group(3))
            keys.append(k)

        keyspace_path = os.path.join(self.restore_dir, keyspace)
        self._delete_old_dir_and_create_new(keyspace_path, tables)
        total_size = reduce(lambda s, k: s + k.size, keys, 0)

        logging.info(
            "Found %(files_count)d files, with total size of %(size)s." %
            dict(files_count=len(keys), size=self._human_size(total_size)))
        print("Found %(files_count)d files, with total size of %(size)s." %
              dict(files_count=len(keys), size=self._human_size(total_size)))

        self._download_keys(keys, total_size)

        if target_hosts and self.run_sstableloader:
            self._run_sstableloader(keyspace_path, tables, target_hosts,
                                    self.cassandra_bin_dir)
        logging.info("Restore completed.")
Ejemplo n.º 2
0
def main():
    subparsers = base_parser.add_subparsers(
        title='subcommands', dest='subcommand')
    base_parser.add_argument(
        '--incremental_backups', action='store_true', default=False)

    put_parser = subparsers.add_parser(
        'put', help="put files on s3 from a manifest")
    manifest_parser = subparsers.add_parser(
        'create-upload-manifest', help="put files on s3 from a manifest")

    # put arguments
    put_parser = add_s3_arguments(put_parser)
    put_parser.add_argument(
        '--bufsize',
        required=False,
        default=BUFFER_SIZE,
        type=int,
        help="Compress and upload buffer size")

    put_parser.add_argument(
        '--manifest',
        required=True,
        help="The manifest containing the files to put on s3")

    put_parser.add_argument(
        '--concurrency',
        required=False,
        default=DEFAULT_CONCURRENCY,
        type=int,
        help="Compress and upload concurrent processes")

    put_parser.add_argument(
        '--reduced-redundancy',
        required=False,
        default=DEFAULT_REDUCED_REDUNDANCY,
        action="store_true",
        help="Compress and upload concurrent processes")

    put_parser.add_argument(
        '--rate-limit',
        required=False,
        default=0,
        type=int,
        help="Limit the upload speed to S3 (by using 'pv'). Value expressed in kilobytes (*1024)")

    # create-upload-manifest arguments
    manifest_parser.add_argument('--snapshot_name', required=True, type=str)
    manifest_parser.add_argument('--conf_path', required=True, type=str)
    manifest_parser.add_argument('--manifest_path', required=True, type=str)
    manifest_parser.add_argument(
        '--snapshot_keyspaces', default='', required=False, type=str)
    manifest_parser.add_argument(
        '--snapshot_table', required=False, default='', type=str)
    manifest_parser.add_argument(
        '--exclude_tables', required=False, type=str)

    args = base_parser.parse_args()
    subcommand = args.subcommand

    if subcommand == 'create-upload-manifest':
        create_upload_manifest(
            args.snapshot_name,
            args.snapshot_keyspaces,
            args.snapshot_table,
            args.conf_path,
            args.manifest_path,
            args.exclude_tables,
            args.incremental_backups
        )

    if subcommand == 'put':
        check_lzop()

        if args.rate_limit > 0:
            check_pv()

        put_from_manifest(
            args.s3_bucket_name,
            get_s3_connection_host(args.s3_bucket_region),
            args.s3_ssenc,
            args.s3_base_path,
            args.aws_access_key_id,
            args.aws_secret_access_key,
            args.manifest,
            args.bufsize,
            args.reduced_redundancy,
            args.rate_limit,
            args.concurrency,
            args.incremental_backups
        )
Ejemplo n.º 3
0
            args.manifest_path,
            args.exclude_tables,
            args.incremental_backups
        )

    if subcommand == 'put':
        check_lzop()

        if args.rate_limit > 0:
            check_pv()

        put_from_manifest(
            args.s3_bucket_name,
            get_s3_connection_host(args.s3_bucket_region),
            args.s3_ssenc,
            args.s3_base_path,
            args.aws_access_key_id,
            args.aws_secret_access_key,
            args.manifest,
            args.bufsize,
            args.reduced_redundancy,
            args.rate_limit,
            args.concurrency,
            args.incremental_backups
        )


if __name__ == '__main__':
    # TODO: if lzop is not available we should fail or run without it
    check_lzop()
Ejemplo n.º 4
0
def main():
    subparsers = base_parser.add_subparsers(
        title='subcommands', dest='subcommand')
    base_parser.add_argument(
        '--incremental_backups', action='store_true', default=False)

    put_parser = subparsers.add_parser(
        'put', help="put files on s3 from a manifest")
    manifest_parser = subparsers.add_parser(
        'create-upload-manifest', help="put files on s3 from a manifest")

    # put arguments
    put_parser = add_s3_arguments(put_parser)
    put_parser.add_argument(
        '--bufsize',
        required=False,
        default=BUFFER_SIZE,
        type=int,
        help="Compress and upload buffer size")

    put_parser.add_argument(
        '--manifest',
        required=True,
        help="The manifest containing the files to put on s3")

    put_parser.add_argument(
        '--concurrency',
        required=False,
        default=DEFAULT_CONCURRENCY,
        type=int,
        help="Compress and upload concurrent processes")

    # create-upload-manifest arguments
    manifest_parser.add_argument('--snapshot_name', required=True, type=str)
    manifest_parser.add_argument('--conf_path', required=True, type=str)
    manifest_parser.add_argument('--manifest_path', required=True, type=str)
    manifest_parser.add_argument(
        '--snapshot_keyspaces', default='', required=False, type=str)
    manifest_parser.add_argument(
        '--snapshot_table', required=False, default='', type=str)

    args = base_parser.parse_args()
    subcommand = args.subcommand

    if subcommand == 'create-upload-manifest':
        create_upload_manifest(
            args.snapshot_name,
            args.snapshot_keyspaces,
            args.snapshot_table,
            args.conf_path,
            args.manifest_path,
            args.incremental_backups
        )

    if subcommand == 'put':
        check_lzop()
        put_from_manifest(
            args.s3_bucket_name,
            get_s3_connection_host(args.s3_bucket_region),
            args.s3_ssenc,
            args.s3_base_path,
            args.aws_access_key_id,
            args.aws_secret_access_key,
            args.manifest,
            args.bufsize,
            args.concurrency,
            args.incremental_backups
        )
Ejemplo n.º 5
0
    if subcommand == 'create-upload-manifest':
        create_upload_manifest(
            args.snapshot_name,
            args.snapshot_keyspaces,
            args.snapshot_table,
            args.conf_path,
            args.manifest_path,
            args.incremental_backups
        )

    if subcommand == 'put':
        check_lzop()
        put_from_manifest(
            args.s3_bucket_name,
            get_s3_connection_host(args.s3_bucket_region),
            args.s3_ssenc,
            args.s3_base_path,
            args.aws_access_key_id,
            args.aws_secret_access_key,
            args.manifest,
            args.bufsize,
            args.concurrency,
            args.incremental_backups
        )


if __name__ == '__main__':
    # TODO: if lzop is not available we should fail or run without it
    check_lzop()
Ejemplo n.º 6
0
def main():
    subparsers = base_parser.add_subparsers(title="subcommands", dest="subcommand")
    base_parser.add_argument("--incremental_backups", action="store_true", default=False)

    put_parser = subparsers.add_parser("put", help="put files on s3 from a manifest")
    manifest_parser = subparsers.add_parser("create-upload-manifest", help="put files on s3 from a manifest")

    # put arguments
    put_parser = add_s3_arguments(put_parser)
    put_parser.add_argument(
        "--bufsize", required=False, default=BUFFER_SIZE, type=int, help="Compress and upload buffer size"
    )

    put_parser.add_argument("--manifest", required=True, help="The manifest containing the files to put on s3")

    put_parser.add_argument(
        "--concurrency",
        required=False,
        default=DEFAULT_CONCURRENCY,
        type=int,
        help="Compress and upload concurrent processes",
    )

    put_parser.add_argument(
        "--reduced-redundancy",
        required=False,
        default=DEFAULT_REDUCED_REDUNDANCY,
        action="store_true",
        help="Compress and upload concurrent processes",
    )

    put_parser.add_argument(
        "--rate-limit",
        required=False,
        default=0,
        type=int,
        help="Limit the upload speed to S3 (by using 'pv'). Value expressed in kilobytes (*1024)",
    )

    # create-upload-manifest arguments
    manifest_parser.add_argument("--snapshot_name", required=True, type=str)
    manifest_parser.add_argument("--conf_path", required=True, type=str)
    manifest_parser.add_argument("--manifest_path", required=True, type=str)
    manifest_parser.add_argument("--snapshot_keyspaces", default="", required=False, type=str)
    manifest_parser.add_argument("--snapshot_table", required=False, default="", type=str)
    manifest_parser.add_argument("--exclude_tables", required=False, type=str)

    args = base_parser.parse_args()
    subcommand = args.subcommand

    if subcommand == "create-upload-manifest":
        create_upload_manifest(
            args.snapshot_name,
            args.snapshot_keyspaces,
            args.snapshot_table,
            args.conf_path,
            args.manifest_path,
            args.exclude_tables,
            args.incremental_backups,
        )

    if subcommand == "put":
        check_lzop()

        if args.rate_limit > 0:
            check_pv()

        put_from_manifest(
            args.s3_bucket_name,
            get_s3_connection_host(args.s3_bucket_region),
            args.s3_ssenc,
            args.s3_base_path,
            args.aws_access_key_id,
            args.aws_secret_access_key,
            args.manifest,
            args.bufsize,
            args.reduced_redundancy,
            args.rate_limit,
            args.concurrency,
            args.incremental_backups,
        )