def test_filter_out_system_keyspaces_if_requested(self): # system tables should be kept by default keep_keyspaces = {} keep_tables = {'k2.t2'} manifest = [ { 'keyspace': 'system', 'columnfamily': 'hints-2666e20573ef38b390fefecf96e8f0c7', 'objects': [] }, { 'keyspace': 'system_distributed', 'columnfamily': 'repair_history', 'objects': [] }, { 'keyspace': 'k2', 'columnfamily': 't2', 'objects': [] }, ] to_restore, ignored = filtering.filter_fqtns(keep_keyspaces, keep_tables, json.dumps(manifest)) self.assertEqual( { 'system.hints-2666e20573ef38b390fefecf96e8f0c7', 'system_distributed.repair_history', 'k2.t2' }, to_restore) self.assertEqual(set(), ignored) # system tables should not be kept if requested keep_keyspaces = {} keep_tables = {'k2.t2'} manifest = [ { 'keyspace': 'system', 'columnfamily': 'hints-2666e20573ef38b390fefecf96e8f0c7', 'objects': [] }, { 'keyspace': 'system_distributed', 'columnfamily': 'repair_history', 'objects': [] }, { 'keyspace': 'k2', 'columnfamily': 't2', 'objects': [] }, ] to_restore, ignored = filtering.filter_fqtns(keep_keyspaces, keep_tables, json.dumps(manifest), True) self.assertEqual({'k2.t2'}, to_restore) self.assertEqual({'system.hints', 'system_distributed.repair_history'}, ignored)
def download_cmd(config, backup_name, download_destination, keyspaces, tables, ignore_system_keyspaces): storage = Storage(config=config.storage) if not download_destination.is_dir(): logging.error('{} is not a directory'.format(download_destination)) sys.exit(1) node_backup = storage.get_node_backup(fqdn=storage.config.fqdn, name=backup_name) if not node_backup.exists(): logging.error('No such backup') sys.exit(1) fqtns_to_download, _ = filter_fqtns(keyspaces, tables, node_backup.manifest, ignore_system_keyspaces) download_data(config.storage, node_backup, fqtns_to_download, download_destination)
def restore_node_sstableloader(config, temp_dir, backup_name, in_place, keep_auth, seeds, storage, keyspaces, tables): cassandra = Cassandra(config) node_backup = None fqdns = config.storage.fqdn.split(",") for fqdn in fqdns: differential_blob = storage.storage_driver.get_blob( os.path.join(fqdn, backup_name, 'meta', 'differential')) node_backup = storage.get_node_backup( fqdn=fqdn, name=backup_name, differential_mode=True if differential_blob is not None else False) if not node_backup.exists(): logging.error('No such backup') sys.exit(1) fqtns_to_restore, ignored_fqtns = filter_fqtns(keyspaces, tables, node_backup.manifest) for fqtns in ignored_fqtns: logging.info('Skipping restore of {}'.format(fqtns)) if len(fqtns_to_restore) == 0: logging.error('There is nothing to restore') sys.exit(0) # Download the backup download_dir = temp_dir / 'medusa-restore-{}'.format(uuid.uuid4()) logging.info('Downloading data from backup to {}'.format(download_dir)) download_data(config.storage, node_backup, fqtns_to_restore, destination=download_dir) invoke_sstableloader(config, download_dir, keep_auth, fqtns_to_restore, cassandra.storage_port) logging.info('Finished loading backup from {}'.format(fqdn)) # Clean the restored data from local temporary folder clean_path(download_dir, keep_folder=False) return node_backup
def test_get_sections_to_restore_with_cfids(self): # kept tables must work also if the manifest has cfids keep_keyspaces = {} keep_tables = {'k2.t2'} manifest = [ { 'keyspace': 'k1', 'columnfamily': 't1-bigBadCfId', 'objects': [] }, { 'keyspace': 'k2', 'columnfamily': 't2-81ffe430e50c11e99f91a15641db358f', 'objects': [] }, ] to_restore, ignored = filtering.filter_fqtns(keep_keyspaces, keep_tables, json.dumps(manifest)) self.assertEqual({'k2.t2-81ffe430e50c11e99f91a15641db358f'}, to_restore) self.assertEqual({'k1.t1'}, ignored)
def restore_node_locally(config, temp_dir, backup_name, in_place, keep_auth, seeds, storage, keyspaces, tables): storage.storage_driver.prepare_download() differential_blob = storage.storage_driver.get_blob( os.path.join(config.storage.fqdn, backup_name, 'meta', 'differential')) node_backup = storage.get_node_backup( fqdn=config.storage.fqdn, name=backup_name, differential_mode=True if differential_blob is not None else False) if not node_backup.exists(): logging.error('No such backup') sys.exit(1) fqtns_to_restore, ignored_fqtns = filter_fqtns(keyspaces, tables, node_backup.manifest) for fqtns in ignored_fqtns: logging.info('Skipping restore of {}'.format(fqtns)) if len(fqtns_to_restore) == 0: logging.error('There is nothing to restore') sys.exit(0) cassandra = Cassandra(config) # Download the backup download_dir = temp_dir / 'medusa-restore-{}'.format(uuid.uuid4()) logging.info('Downloading data from backup to {}'.format(download_dir)) download_data(config.storage, node_backup, fqtns_to_restore, destination=download_dir) if not medusa.utils.evaluate_boolean(config.kubernetes.enabled): logging.info('Stopping Cassandra') cassandra.shutdown() wait_for_node_to_go_down(config, cassandra.hostname) # Clean the commitlogs, the saved cache to prevent any kind of conflict # especially around system tables. use_sudo = not medusa.utils.evaluate_boolean(config.kubernetes.enabled) clean_path(cassandra.commit_logs_path, use_sudo, keep_folder=True) clean_path(cassandra.saved_caches_path, use_sudo, keep_folder=True) # move backup data to Cassandra data directory according to system table logging.info('Moving backup data to Cassandra data directory') manifest = json.loads(node_backup.manifest) for section in manifest: fqtn = "{}.{}".format(section['keyspace'], section['columnfamily']) if fqtn not in fqtns_to_restore: logging.debug('Skipping restore for {}'.format(fqtn)) continue maybe_restore_section(section, download_dir, cassandra.root, in_place, keep_auth, use_sudo) node_fqdn = storage.config.fqdn token_map_file = download_dir / 'tokenmap.json' with open(str(token_map_file), 'r') as f: tokens = get_node_tokens(node_fqdn, f) logging.debug("Parsed tokens: {}".format(tokens)) # possibly wait for seeds # # In a Kubernetes deployment we can assume that seed nodes will be started first. It will # handled either by the statefulset controller or by the controller of a Cassandra # operator. if not medusa.utils.evaluate_boolean(config.kubernetes.enabled): if seeds is not None: wait_for_seeds(config, seeds) else: logging.info('No --seeds specified so we will not wait for any') # Start up Cassandra logging.info('Starting Cassandra') # restoring in place retains system.local, which has tokens in it. no need to specify extra if in_place: cassandra.start_with_implicit_token() else: cassandra.start(tokens) # Clean the restored data from local temporary folder clean_path(download_dir, use_sudo, keep_folder=False) return node_backup
def test_get_sections_to_restore(self): # nothing skipped, both tables make it to restore keep_keyspaces = {} keep_tables = {} manifest = [{ 'keyspace': 'k1', 'columnfamily': 't1', 'objects': [] }, { 'keyspace': 'k2', 'columnfamily': 't2', 'objects': [] }] to_restore, ignored = filtering.filter_fqtns(keep_keyspaces, keep_tables, json.dumps(manifest)) self.assertEqual({'k1.t1', 'k2.t2'}, to_restore) self.assertEqual(set(), ignored) # skipping one table (must be specified as a fqtn) keep_keyspaces = {} keep_tables = {'k2.t2'} manifest = [{ 'keyspace': 'k1', 'columnfamily': 't1', 'objects': [] }, { 'keyspace': 'k2', 'columnfamily': 't2', 'objects': [] }] to_restore, ignored = filtering.filter_fqtns(keep_keyspaces, keep_tables, json.dumps(manifest)) self.assertEqual({'k2.t2'}, to_restore) self.assertEqual({'k1.t1'}, ignored) # saying only table name doesn't cause a keep keep_keyspaces = {} keep_tables = {'t2'} manifest = [{ 'keyspace': 'k1', 'columnfamily': 't1', 'objects': [] }, { 'keyspace': 'k2', 'columnfamily': 't2', 'objects': [] }] to_restore, ignored = filtering.filter_fqtns(keep_keyspaces, keep_tables, json.dumps(manifest)) self.assertEqual(set(), to_restore) self.assertEqual({'k1.t1', 'k2.t2'}, ignored) # keeping the whole keyspace keep_keyspaces = {'k2'} keep_tables = {} manifest = [{ 'keyspace': 'k1', 'columnfamily': 't1', 'objects': [] }, { 'keyspace': 'k2', 'columnfamily': 't2', 'objects': [] }, { 'keyspace': 'k2', 'columnfamily': 't3', 'objects': [] }] to_restore, ignored = filtering.filter_fqtns(keep_keyspaces, keep_tables, json.dumps(manifest)) self.assertEqual({'k2.t2', 'k2.t3'}, to_restore) self.assertEqual({'k1.t1'}, ignored)