def _i_can_download_the_backup_single_table_successfully( context, backup_name, fqtn): def cleanup(temp_path): if os.path.exists(temp_path) and os.path.isdir(temp_path): shutil.rmtree(temp_path) storage = Storage(config=context.medusa_config.storage) config = context.medusa_config download_path = os.path.join("/tmp", "medusa-download-one-table/") cleanup(download_path) os.makedirs(download_path) backup = storage.get_node_backup( fqdn=config.storage.fqdn, name=backup_name, ) # download_data requires fqtn with table id fqtns_to_download, _ = medusa.filtering.filter_fqtns([], [fqtn], backup.manifest, True) medusa.download.download_data(context.medusa_config.storage, backup, fqtns_to_download, Path(download_path)) # check the keyspace directory has been created ks, table = fqtn.split('.') ks_path = os.path.join(download_path, ks) assert os.path.isdir(ks_path) # check tables have been downloaded assert list(Path(ks_path).glob('{}-*/*.db'.format(table))) cleanup(download_path)
def _i_can_download_the_backup_all_tables_successfully(context, backup_name): def cleanup(temp_path): if os.path.exists(temp_path) and os.path.isdir(temp_path): shutil.rmtree(temp_path) storage = Storage(config=context.medusa_config.storage) config = context.medusa_config download_path = os.path.join("/tmp", "medusa-download-all-tables/") cleanup(download_path) os.makedirs(download_path) backup = storage.get_node_backup( fqdn=config.storage.fqdn, name=backup_name, ) fqtn = set({}) medusa.download.download_data(context.medusa_config.storage, backup, fqtn, Path(download_path)) # check all manifest objects that have been backed up have been downloaded keyspaces = { section['keyspace'] for section in json.loads(backup.manifest) if section['objects'] } for ks in keyspaces: ks_path = os.path.join(download_path, ks) assert os.path.isdir(ks_path) cleanup(download_path)
def download_cmd(config, backup_name, download_destination): storage = Storage(config=config.storage) if not download_destination.is_dir(): logging.error('{} is not a directory'.format(download_destination)) sys.exit(1) node_backup = storage.get_node_backup(fqdn=storage.config.fqdn, name=backup_name) if not node_backup.exists(): logging.error('No such backup') sys.exit(1) download_data(config.storage, node_backup, download_destination)
def handle_backup(config, backup_name_arg, stagger_time, enable_md5_checks_flag, mode): start = datetime.datetime.now() backup_name = backup_name_arg or start.strftime('%Y%m%d%H%M') monitoring = Monitoring(config=config.monitoring) try: logging.debug( "Starting backup preparations with Mode: {}".format(mode)) storage = Storage(config=config.storage) cassandra = Cassandra(config) differential_mode = False if mode == "differential": differential_mode = True node_backup = storage.get_node_backup( fqdn=config.storage.fqdn, name=backup_name, differential_mode=differential_mode) if node_backup.exists(): raise IOError( 'Error: Backup {} already exists'.format(backup_name)) # Starting the backup logging.info( "Starting backup using Stagger: {} Mode: {} Name: {}".format( stagger_time, mode, backup_name)) BackupMan.update_backup_status(backup_name, BackupMan.STATUS_IN_PROGRESS) info = start_backup(storage, node_backup, cassandra, differential_mode, stagger_time, start, mode, enable_md5_checks_flag, backup_name, config, monitoring) BackupMan.update_backup_status(backup_name, BackupMan.STATUS_SUCCESS) logging.debug("Done with backup, returning backup result information") return (info["actual_backup_duration"], info["actual_start_time"], info["end_time"], info["node_backup"], info["node_backup_cache"], info["num_files"], info["start_time"], info["backup_name"]) except Exception as e: logging.error( "Issue occurred inside handle_backup Name: {} Error: {}".format( backup_name, str(e))) BackupMan.update_backup_status(backup_name, BackupMan.STATUS_FAILED) tags = ['medusa-node-backup', 'backup-error', backup_name] monitoring.send(tags, 1) medusa.utils.handle_exception( e, "Error occurred during backup: {}".format(str(e)), config)
def download_cmd(config, backup_name, download_destination, keyspaces, tables, ignore_system_keyspaces): storage = Storage(config=config.storage) if not download_destination.is_dir(): logging.error('{} is not a directory'.format(download_destination)) sys.exit(1) node_backup = storage.get_node_backup(fqdn=storage.config.fqdn, name=backup_name) if not node_backup.exists(): logging.error('No such backup') sys.exit(1) fqtns_to_download, _ = filter_fqtns(keyspaces, tables, node_backup.manifest, ignore_system_keyspaces) download_data(config.storage, node_backup, fqtns_to_download, download_destination)
def _i_can_download_the_backup_single_table_successfully( context, backup_name, fqtn): def cleanup(temp_path): if os.path.exists(temp_path) and os.path.isdir(temp_path): shutil.rmtree(temp_path) storage = Storage(config=context.medusa_config.storage) config = context.medusa_config download_path = os.path.join("/tmp", "medusa-download-one-table/") cleanup(download_path) os.makedirs(download_path) backup = storage.get_node_backup( fqdn=config.storage.fqdn, name=backup_name, ) medusa.download.download_data(context.medusa_config.storage, backup, fqtn, Path(download_path)) sys_dist = os.path.join(download_path, 'system_distributed') assert os.path.isdir(sys_dist) cleanup(download_path)
def main(config, backup_name_arg, stagger_time, mode): start = datetime.datetime.now() backup_name = backup_name_arg or start.strftime('%Y%m%d%H') monitoring = Monitoring(config=config.monitoring) try: storage = Storage(config=config.storage) cassandra = Cassandra(config) differential_mode = False if mode == "differential": differential_mode = True node_backup = storage.get_node_backup( fqdn=config.storage.fqdn, name=backup_name, differential_mode=differential_mode ) if node_backup.exists(): raise IOError('Error: Backup {} already exists'.format(backup_name)) # Make sure that priority remains to Cassandra/limiting backups resource usage try: throttle_backup() except Exception: logging.warning("Throttling backup impossible. It's probable that ionice is not available.") logging.info('Saving tokenmap and schema') schema, tokenmap = get_schema_and_tokenmap(cassandra) node_backup.schema = schema node_backup.tokenmap = json.dumps(tokenmap) if differential_mode is True: node_backup.differential = mode add_backup_start_to_index(storage, node_backup) if stagger_time: stagger_end = start + stagger_time logging.info('Staggering backup run, trying until {}'.format(stagger_end)) while not stagger(config.storage.fqdn, storage, tokenmap): if datetime.datetime.now() < stagger_end: logging.info('Staggering this backup run...') time.sleep(60) else: raise IOError('Backups on previous nodes did not complete' ' within our stagger time.') actual_start = datetime.datetime.now() num_files, node_backup_cache = do_backup( cassandra, node_backup, storage, differential_mode, config, backup_name) end = datetime.datetime.now() actual_backup_duration = end - actual_start print_backup_stats(actual_backup_duration, actual_start, end, node_backup, node_backup_cache, num_files, start) update_monitoring(actual_backup_duration, backup_name, monitoring, node_backup) return (actual_backup_duration, actual_start, end, node_backup, node_backup_cache, num_files, start) except Exception as e: tags = ['medusa-node-backup', 'backup-error', backup_name] monitoring.send(tags, 1) medusa.utils.handle_exception( e, "This error happened during the backup: {}".format(str(e)), config )