def _i_can_download_the_backup_single_table_successfully(
context, backup_name, fqtn):
def cleanup(temp_path):
if os.path.exists(temp_path) and os.path.isdir(temp_path):
shutil.rmtree(temp_path)
storage = Storage(config=context.medusa_config.storage)
config = context.medusa_config
download_path = os.path.join("/tmp", "medusa-download-one-table/")
cleanup(download_path)
os.makedirs(download_path)
backup = storage.get_node_backup(
fqdn=config.storage.fqdn,
name=backup_name,
)
# download_data requires fqtn with table id
fqtns_to_download, _ = medusa.filtering.filter_fqtns([], [fqtn],
backup.manifest, True)
medusa.download.download_data(context.medusa_config.storage, backup,
fqtns_to_download, Path(download_path))
# check the keyspace directory has been created
ks, table = fqtn.split('.')
ks_path = os.path.join(download_path, ks)
assert os.path.isdir(ks_path)
# check tables have been downloaded
assert list(Path(ks_path).glob('{}-*/*.db'.format(table)))
cleanup(download_path)
def _i_can_download_the_backup_all_tables_successfully(context, backup_name):
def cleanup(temp_path):
if os.path.exists(temp_path) and os.path.isdir(temp_path):
shutil.rmtree(temp_path)
storage = Storage(config=context.medusa_config.storage)
config = context.medusa_config
download_path = os.path.join("/tmp", "medusa-download-all-tables/")
cleanup(download_path)
os.makedirs(download_path)
backup = storage.get_node_backup(
fqdn=config.storage.fqdn,
name=backup_name,
)
fqtn = set({})
medusa.download.download_data(context.medusa_config.storage, backup, fqtn,
Path(download_path))
# check all manifest objects that have been backed up have been downloaded
keyspaces = {
section['keyspace']
for section in json.loads(backup.manifest) if section['objects']
}
for ks in keyspaces:
ks_path = os.path.join(download_path, ks)
assert os.path.isdir(ks_path)
cleanup(download_path)
def download_cmd(config, backup_name, download_destination):
storage = Storage(config=config.storage)
if not download_destination.is_dir():
logging.error('{} is not a directory'.format(download_destination))
sys.exit(1)
node_backup = storage.get_node_backup(fqdn=storage.config.fqdn, name=backup_name)
if not node_backup.exists():
logging.error('No such backup')
sys.exit(1)
download_data(config.storage, node_backup, download_destination)
def handle_backup(config, backup_name_arg, stagger_time,
enable_md5_checks_flag, mode):
start = datetime.datetime.now()
backup_name = backup_name_arg or start.strftime('%Y%m%d%H%M')
monitoring = Monitoring(config=config.monitoring)
try:
logging.debug(
"Starting backup preparations with Mode: {}".format(mode))
storage = Storage(config=config.storage)
cassandra = Cassandra(config)
differential_mode = False
if mode == "differential":
differential_mode = True
node_backup = storage.get_node_backup(
fqdn=config.storage.fqdn,
name=backup_name,
differential_mode=differential_mode)
if node_backup.exists():
raise IOError(
'Error: Backup {} already exists'.format(backup_name))
# Starting the backup
logging.info(
"Starting backup using Stagger: {} Mode: {} Name: {}".format(
stagger_time, mode, backup_name))
BackupMan.update_backup_status(backup_name,
BackupMan.STATUS_IN_PROGRESS)
info = start_backup(storage, node_backup, cassandra, differential_mode,
stagger_time, start, mode, enable_md5_checks_flag,
backup_name, config, monitoring)
BackupMan.update_backup_status(backup_name, BackupMan.STATUS_SUCCESS)
logging.debug("Done with backup, returning backup result information")
return (info["actual_backup_duration"], info["actual_start_time"],
info["end_time"], info["node_backup"],
info["node_backup_cache"], info["num_files"],
info["start_time"], info["backup_name"])
except Exception as e:
logging.error(
"Issue occurred inside handle_backup Name: {} Error: {}".format(
backup_name, str(e)))
BackupMan.update_backup_status(backup_name, BackupMan.STATUS_FAILED)
tags = ['medusa-node-backup', 'backup-error', backup_name]
monitoring.send(tags, 1)
medusa.utils.handle_exception(
e, "Error occurred during backup: {}".format(str(e)), config)
def download_cmd(config, backup_name, download_destination, keyspaces, tables,
ignore_system_keyspaces):
storage = Storage(config=config.storage)
if not download_destination.is_dir():
logging.error('{} is not a directory'.format(download_destination))
sys.exit(1)
node_backup = storage.get_node_backup(fqdn=storage.config.fqdn,
name=backup_name)
if not node_backup.exists():
logging.error('No such backup')
sys.exit(1)
fqtns_to_download, _ = filter_fqtns(keyspaces, tables,
node_backup.manifest,
ignore_system_keyspaces)
download_data(config.storage, node_backup, fqtns_to_download,
download_destination)
def _i_can_download_the_backup_single_table_successfully(
context, backup_name, fqtn):
def cleanup(temp_path):
if os.path.exists(temp_path) and os.path.isdir(temp_path):
shutil.rmtree(temp_path)
storage = Storage(config=context.medusa_config.storage)
config = context.medusa_config
download_path = os.path.join("/tmp", "medusa-download-one-table/")
cleanup(download_path)
os.makedirs(download_path)
backup = storage.get_node_backup(
fqdn=config.storage.fqdn,
name=backup_name,
)
medusa.download.download_data(context.medusa_config.storage, backup, fqtn,
Path(download_path))
sys_dist = os.path.join(download_path, 'system_distributed')
assert os.path.isdir(sys_dist)
cleanup(download_path)
def main(config, backup_name_arg, stagger_time, mode):
start = datetime.datetime.now()
backup_name = backup_name_arg or start.strftime('%Y%m%d%H')
monitoring = Monitoring(config=config.monitoring)
try:
storage = Storage(config=config.storage)
cassandra = Cassandra(config)
differential_mode = False
if mode == "differential":
differential_mode = True
node_backup = storage.get_node_backup(
fqdn=config.storage.fqdn,
name=backup_name,
differential_mode=differential_mode
)
if node_backup.exists():
raise IOError('Error: Backup {} already exists'.format(backup_name))
# Make sure that priority remains to Cassandra/limiting backups resource usage
try:
throttle_backup()
except Exception:
logging.warning("Throttling backup impossible. It's probable that ionice is not available.")
logging.info('Saving tokenmap and schema')
schema, tokenmap = get_schema_and_tokenmap(cassandra)
node_backup.schema = schema
node_backup.tokenmap = json.dumps(tokenmap)
if differential_mode is True:
node_backup.differential = mode
add_backup_start_to_index(storage, node_backup)
if stagger_time:
stagger_end = start + stagger_time
logging.info('Staggering backup run, trying until {}'.format(stagger_end))
while not stagger(config.storage.fqdn, storage, tokenmap):
if datetime.datetime.now() < stagger_end:
logging.info('Staggering this backup run...')
time.sleep(60)
else:
raise IOError('Backups on previous nodes did not complete'
' within our stagger time.')
actual_start = datetime.datetime.now()
num_files, node_backup_cache = do_backup(
cassandra, node_backup, storage, differential_mode, config, backup_name)
end = datetime.datetime.now()
actual_backup_duration = end - actual_start
print_backup_stats(actual_backup_duration, actual_start, end, node_backup, node_backup_cache, num_files, start)
update_monitoring(actual_backup_duration, backup_name, monitoring, node_backup)
return (actual_backup_duration, actual_start, end, node_backup, node_backup_cache, num_files, start)
except Exception as e:
tags = ['medusa-node-backup', 'backup-error', backup_name]
monitoring.send(tags, 1)
medusa.utils.handle_exception(
e,
"This error happened during the backup: {}".format(str(e)),
config
)
