def test_filter_out_system_keyspaces_if_requested(self):
# system tables should be kept by default
keep_keyspaces = {}
keep_tables = {'k2.t2'}
manifest = [
{
'keyspace': 'system',
'columnfamily': 'hints-2666e20573ef38b390fefecf96e8f0c7',
'objects': []
},
{
'keyspace': 'system_distributed',
'columnfamily': 'repair_history',
'objects': []
},
{
'keyspace': 'k2',
'columnfamily': 't2',
'objects': []
},
]
to_restore, ignored = filtering.filter_fqtns(keep_keyspaces,
keep_tables,
json.dumps(manifest))
self.assertEqual(
{
'system.hints-2666e20573ef38b390fefecf96e8f0c7',
'system_distributed.repair_history', 'k2.t2'
}, to_restore)
self.assertEqual(set(), ignored)
# system tables should not be kept if requested
keep_keyspaces = {}
keep_tables = {'k2.t2'}
manifest = [
{
'keyspace': 'system',
'columnfamily': 'hints-2666e20573ef38b390fefecf96e8f0c7',
'objects': []
},
{
'keyspace': 'system_distributed',
'columnfamily': 'repair_history',
'objects': []
},
{
'keyspace': 'k2',
'columnfamily': 't2',
'objects': []
},
]
to_restore, ignored = filtering.filter_fqtns(keep_keyspaces,
keep_tables,
json.dumps(manifest),
True)
self.assertEqual({'k2.t2'}, to_restore)
self.assertEqual({'system.hints', 'system_distributed.repair_history'},
ignored)
def download_cmd(config, backup_name, download_destination, keyspaces, tables,
ignore_system_keyspaces):
storage = Storage(config=config.storage)
if not download_destination.is_dir():
logging.error('{} is not a directory'.format(download_destination))
sys.exit(1)
node_backup = storage.get_node_backup(fqdn=storage.config.fqdn,
name=backup_name)
if not node_backup.exists():
logging.error('No such backup')
sys.exit(1)
fqtns_to_download, _ = filter_fqtns(keyspaces, tables,
node_backup.manifest,
ignore_system_keyspaces)
download_data(config.storage, node_backup, fqtns_to_download,
download_destination)
def restore_node_sstableloader(config, temp_dir, backup_name, in_place,
keep_auth, seeds, storage, keyspaces, tables):
cassandra = Cassandra(config)
node_backup = None
fqdns = config.storage.fqdn.split(",")
for fqdn in fqdns:
differential_blob = storage.storage_driver.get_blob(
os.path.join(fqdn, backup_name, 'meta', 'differential'))
node_backup = storage.get_node_backup(
fqdn=fqdn,
name=backup_name,
differential_mode=True if differential_blob is not None else False)
if not node_backup.exists():
logging.error('No such backup')
sys.exit(1)
fqtns_to_restore, ignored_fqtns = filter_fqtns(keyspaces, tables,
node_backup.manifest)
for fqtns in ignored_fqtns:
logging.info('Skipping restore of {}'.format(fqtns))
if len(fqtns_to_restore) == 0:
logging.error('There is nothing to restore')
sys.exit(0)
# Download the backup
download_dir = temp_dir / 'medusa-restore-{}'.format(uuid.uuid4())
logging.info('Downloading data from backup to {}'.format(download_dir))
download_data(config.storage,
node_backup,
fqtns_to_restore,
destination=download_dir)
invoke_sstableloader(config, download_dir, keep_auth, fqtns_to_restore,
cassandra.storage_port)
logging.info('Finished loading backup from {}'.format(fqdn))
# Clean the restored data from local temporary folder
clean_path(download_dir, keep_folder=False)
return node_backup
def test_get_sections_to_restore_with_cfids(self):
# kept tables must work also if the manifest has cfids
keep_keyspaces = {}
keep_tables = {'k2.t2'}
manifest = [
{
'keyspace': 'k1',
'columnfamily': 't1-bigBadCfId',
'objects': []
},
{
'keyspace': 'k2',
'columnfamily': 't2-81ffe430e50c11e99f91a15641db358f',
'objects': []
},
]
to_restore, ignored = filtering.filter_fqtns(keep_keyspaces,
keep_tables,
json.dumps(manifest))
self.assertEqual({'k2.t2-81ffe430e50c11e99f91a15641db358f'},
to_restore)
self.assertEqual({'k1.t1'}, ignored)
def restore_node_locally(config, temp_dir, backup_name, in_place, keep_auth,
seeds, storage, keyspaces, tables):
storage.storage_driver.prepare_download()
differential_blob = storage.storage_driver.get_blob(
os.path.join(config.storage.fqdn, backup_name, 'meta', 'differential'))
node_backup = storage.get_node_backup(
fqdn=config.storage.fqdn,
name=backup_name,
differential_mode=True if differential_blob is not None else False)
if not node_backup.exists():
logging.error('No such backup')
sys.exit(1)
fqtns_to_restore, ignored_fqtns = filter_fqtns(keyspaces, tables,
node_backup.manifest)
for fqtns in ignored_fqtns:
logging.info('Skipping restore of {}'.format(fqtns))
if len(fqtns_to_restore) == 0:
logging.error('There is nothing to restore')
sys.exit(0)
cassandra = Cassandra(config)
# Download the backup
download_dir = temp_dir / 'medusa-restore-{}'.format(uuid.uuid4())
logging.info('Downloading data from backup to {}'.format(download_dir))
download_data(config.storage,
node_backup,
fqtns_to_restore,
destination=download_dir)
if not medusa.utils.evaluate_boolean(config.kubernetes.enabled):
logging.info('Stopping Cassandra')
cassandra.shutdown()
wait_for_node_to_go_down(config, cassandra.hostname)
# Clean the commitlogs, the saved cache to prevent any kind of conflict
# especially around system tables.
use_sudo = not medusa.utils.evaluate_boolean(config.kubernetes.enabled)
clean_path(cassandra.commit_logs_path, use_sudo, keep_folder=True)
clean_path(cassandra.saved_caches_path, use_sudo, keep_folder=True)
# move backup data to Cassandra data directory according to system table
logging.info('Moving backup data to Cassandra data directory')
manifest = json.loads(node_backup.manifest)
for section in manifest:
fqtn = "{}.{}".format(section['keyspace'], section['columnfamily'])
if fqtn not in fqtns_to_restore:
logging.debug('Skipping restore for {}'.format(fqtn))
continue
maybe_restore_section(section, download_dir, cassandra.root, in_place,
keep_auth, use_sudo)
node_fqdn = storage.config.fqdn
token_map_file = download_dir / 'tokenmap.json'
with open(str(token_map_file), 'r') as f:
tokens = get_node_tokens(node_fqdn, f)
logging.debug("Parsed tokens: {}".format(tokens))
# possibly wait for seeds
#
# In a Kubernetes deployment we can assume that seed nodes will be started first. It will
# handled either by the statefulset controller or by the controller of a Cassandra
# operator.
if not medusa.utils.evaluate_boolean(config.kubernetes.enabled):
if seeds is not None:
wait_for_seeds(config, seeds)
else:
logging.info('No --seeds specified so we will not wait for any')
# Start up Cassandra
logging.info('Starting Cassandra')
# restoring in place retains system.local, which has tokens in it. no need to specify extra
if in_place:
cassandra.start_with_implicit_token()
else:
cassandra.start(tokens)
# Clean the restored data from local temporary folder
clean_path(download_dir, use_sudo, keep_folder=False)
return node_backup
def test_get_sections_to_restore(self):
# nothing skipped, both tables make it to restore
keep_keyspaces = {}
keep_tables = {}
manifest = [{
'keyspace': 'k1',
'columnfamily': 't1',
'objects': []
}, {
'keyspace': 'k2',
'columnfamily': 't2',
'objects': []
}]
to_restore, ignored = filtering.filter_fqtns(keep_keyspaces,
keep_tables,
json.dumps(manifest))
self.assertEqual({'k1.t1', 'k2.t2'}, to_restore)
self.assertEqual(set(), ignored)
# skipping one table (must be specified as a fqtn)
keep_keyspaces = {}
keep_tables = {'k2.t2'}
manifest = [{
'keyspace': 'k1',
'columnfamily': 't1',
'objects': []
}, {
'keyspace': 'k2',
'columnfamily': 't2',
'objects': []
}]
to_restore, ignored = filtering.filter_fqtns(keep_keyspaces,
keep_tables,
json.dumps(manifest))
self.assertEqual({'k2.t2'}, to_restore)
self.assertEqual({'k1.t1'}, ignored)
# saying only table name doesn't cause a keep
keep_keyspaces = {}
keep_tables = {'t2'}
manifest = [{
'keyspace': 'k1',
'columnfamily': 't1',
'objects': []
}, {
'keyspace': 'k2',
'columnfamily': 't2',
'objects': []
}]
to_restore, ignored = filtering.filter_fqtns(keep_keyspaces,
keep_tables,
json.dumps(manifest))
self.assertEqual(set(), to_restore)
self.assertEqual({'k1.t1', 'k2.t2'}, ignored)
# keeping the whole keyspace
keep_keyspaces = {'k2'}
keep_tables = {}
manifest = [{
'keyspace': 'k1',
'columnfamily': 't1',
'objects': []
}, {
'keyspace': 'k2',
'columnfamily': 't2',
'objects': []
}, {
'keyspace': 'k2',
'columnfamily': 't3',
'objects': []
}]
to_restore, ignored = filtering.filter_fqtns(keep_keyspaces,
keep_tables,
json.dumps(manifest))
self.assertEqual({'k2.t2', 'k2.t3'}, to_restore)
self.assertEqual({'k1.t1'}, ignored)