def main(config, max_backup_age=0, max_backup_count=0):
backups_to_purge = set()
monitoring = Monitoring(config=config.monitoring)
try:
logging.info('Starting purge')
storage = Storage(config=config.storage)
# Get all backups for the local node
logging.info('Listing backups for {}'.format(config.storage.fqdn))
backup_index = storage.list_backup_index_blobs()
backups = list(
storage.list_node_backups(fqdn=config.storage.fqdn,
backup_index_blobs=backup_index))
# list all backups to purge based on date conditions
backups_to_purge |= set(
backups_to_purge_by_age(backups, max_backup_age))
# list all backups to purge based on count conditions
backups_to_purge |= set(
backups_to_purge_by_count(backups, max_backup_count))
# purge all candidate backups
purge_backups(storage, backups_to_purge)
logging.debug('Emitting metrics')
tags = ['medusa-node-backup', 'purge-error', 'PURGE-ERROR']
monitoring.send(tags, 0)
except Exception as e:
traceback.print_exc()
tags = ['medusa-node-backup', 'purge-error', 'PURGE-ERROR']
monitoring.send(tags, 1)
logging.error('This error happened during the purge: {}'.format(
str(e)))
sys.exit(1)
def _i_can_see_secondary_index_files_in_backup(context, backup_name):
storage = Storage(config=context.medusa_config.storage)
node_backups = storage.list_node_backups()
target_backup = list(filter(lambda backup: backup.name == backup_name, node_backups))[0]
manifest = json.loads(target_backup.manifest)
seen_index_files = 0
for section in manifest:
for f in section['objects']:
if 'idx' in f['path']:
seen_index_files += 1
assert seen_index_files > 0
def _backup_named_something_has_nb_files_in_the_manifest(context, backup_name, nb_files, table_name, keyspace_name):
storage = Storage(config=context.medusa_config.storage)
node_backups = storage.list_node_backups()
# Find the backup we're looking for
target_backup = list(filter(lambda backup: backup.name == backup_name, node_backups))[0]
# Parse its manifest
manifest = json.loads(target_backup.manifest)
for section in manifest:
if section['keyspace'] == keyspace_name and section['columnfamily'][:len(table_name)] == table_name:
if len(section['objects']) != int(nb_files):
logging.error("Was expecting {} files, got {}".format(nb_files, len(section['objects'])))
assert len(section['objects']) == int(nb_files)
def _i_can_see_purged_backup_files_for_the_tablename_table_in_keyspace_keyspacename(
context, can_see_purged, table_name, keyspace):
storage = Storage(config=context.medusa_config.storage)
path = os.path.join(
storage.prefix_path + context.medusa_config.storage.fqdn, "data",
keyspace, table_name)
sb_files = len(storage.storage_driver.list_objects(path))
node_backups = storage.list_node_backups()
# Parse its manifest
nb_list = list(node_backups)
nb_files = {}
for nb in nb_list:
manifest = json.loads(nb.manifest)
for section in manifest:
if (section["keyspace"] == keyspace and
section["columnfamily"][:len(table_name)] == table_name):
for objects in section["objects"]:
nb_files[objects["path"]] = 0
if can_see_purged == "not":
assert sb_files == len(nb_files)
else:
# GC grace is activated and we expect more files in the storage bucket than in the manifests
assert sb_files > len(nb_files)
