def _the_backup_index_exists(context):
storage = Storage(config=context.medusa_config.storage)
assert True is medusa.index.index_exists(storage)
class RestoreNodeTest(unittest.TestCase):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.local_storage_dir = "/tmp/medusa_local_storage"
self.medusa_bucket_dir = "/tmp/medusa_test_bucket"
def setUp(self):
if os.path.isdir(self.local_storage_dir):
shutil.rmtree(self.local_storage_dir)
if os.path.isdir(self.medusa_bucket_dir):
shutil.rmtree(self.medusa_bucket_dir)
os.makedirs(self.local_storage_dir)
config = configparser.ConfigParser(interpolation=None)
config['storage'] = {
'host_file_separator': ',',
'bucket_name': 'medusa_test_bucket',
'key_file': '',
'storage_provider': 'local',
'prefix': '',
'fqdn': '127.0.0.1',
'api_key_or_username': '',
'api_secret_or_password': '',
'base_path': '/tmp'
}
config['cassandra'] = {'is_ccm': 1}
self.config = MedusaConfig(
storage=_namedtuple_from_dict(StorageConfig, config['storage']),
cassandra=_namedtuple_from_dict(CassandraConfig,
config['cassandra']),
monitoring={},
ssh=None,
restore=None)
self.storage = Storage(config=self.config.storage)
def test_add_object_from_string(self):
file_content = "content of the test file"
self.storage.storage_driver.upload_blob_from_string(
"test1/file.txt", file_content)
self.assertEquals(
self.storage.storage_driver.get_blob_content_as_string(
"test1/file.txt"), file_content)
def test_download_blobs(self):
files_to_download = list()
file1_content = "content of the test file1"
file2_content = "content of the test file2"
self.storage.storage_driver.upload_blob_from_string(
"test_download_blobs1/file1.txt", file1_content)
files_to_download.append("test_download_blobs1/file1.txt")
self.storage.storage_driver.upload_blob_from_string(
"test_download_blobs2/file2.txt", file2_content)
files_to_download.append("test_download_blobs2/file2.txt")
self.assertEquals(len(os.listdir(self.medusa_bucket_dir)), 2)
self.storage.storage_driver.download_blobs(files_to_download,
self.local_storage_dir)
self.assertEquals(len(os.listdir(self.local_storage_dir)), 2)
def test_list_objects(self):
file1_content = "content of the test file1"
file2_content = "content of the test file2"
self.storage.storage_driver.upload_blob_from_string(
"test_download_blobs1/file1.txt", file1_content)
self.storage.storage_driver.upload_blob_from_string(
"test_download_blobs2/file2.txt", file2_content)
objects = self.storage.storage_driver.list_objects()
self.assertEquals(len(objects), 2)
one_object = self.storage.storage_driver.list_objects(
"test_download_blobs2")
self.assertEquals(len(one_object), 1)
def test_read_blob(self):
file1_content = "content of the test file1"
self.storage.storage_driver.upload_blob_from_string(
"test_download_blobs1/file1.txt", file1_content)
objects = self.storage.storage_driver.list_objects(
"test_download_blobs1")
object_content = self.storage.storage_driver.read_blob_as_string(
objects[0])
self.assertEquals(object_content, file1_content)
def test_get_blob(self):
file1_content = "content of the test file1"
self.storage.storage_driver.upload_blob_from_string(
"test_download_blobs1/file1.txt", file1_content)
obj = self.storage.storage_driver.get_blob(
"test_download_blobs1/file1.txt")
self.assertEquals(obj.name, "test_download_blobs1/file1.txt")
def test_read_blob_as_bytes(self):
file1_content = "content of the test file1"
self.storage.storage_driver.upload_blob_from_string(
"test_download_blobs1/file1.txt", file1_content)
object_content = self.storage.storage_driver.get_blob_content_as_bytes(
"test_download_blobs1/file1.txt")
self.assertEquals(object_content, b"content of the test file1")
def test_verify_hash(self):
file1_content = "content of the test file1"
manifest = self.storage.storage_driver.upload_blob_from_string(
"test_download_blobs1/file1.txt", file1_content)
obj = self.storage.storage_driver.get_blob(
"test_download_blobs1/file1.txt")
self.assertEquals(manifest.MD5, obj.hash)
def test_hashes_match(self):
# Should match
hash1 = "S1EAM/BVMqhbJnAUs/nWlQ=="
hash2 = "4b510033f05532a85b267014b3f9d695"
self.assertTrue(
medusa.storage.abstract_storage.AbstractStorage.hashes_match(
hash1, hash2))
# Should match
hash1 = "4b510033f05532a85b267014b3f9d695"
hash2 = "4b510033f05532a85b267014b3f9d695"
self.assertTrue(
medusa.storage.abstract_storage.AbstractStorage.hashes_match(
hash1, hash2))
# Should not match
hash1 = "S1EAM/BVMqhbJnAUs/nWlQsdfsdf=="
hash2 = "4b510033f05532a85b267014b3f9d695"
self.assertFalse(
medusa.storage.abstract_storage.AbstractStorage.hashes_match(
hash1, hash2))
def test_generate_md5_hash(self):
with tempfile.NamedTemporaryFile() as tf:
# write random bytes
two_megabytes = 2 * 1024 * 1024
tf.write(os.urandom(two_megabytes))
tf.flush()
# compute checksum of the whole file at once
tf.seek(0)
checksum_full = hashlib.md5(tf.read()).digest()
digest_full = base64.encodestring(checksum_full).decode(
'UTF-8').strip()
# compute checksum using default-size chunks
tf.seek(0)
digest_chunk = generate_md5_hash(tf.name)
# compare the digests
self.assertEqual(digest_chunk, digest_full)
# compute checksum using custom size chunks
tf.seek(0)
self.assertEqual(digest_full,
generate_md5_hash(tf.name, block_size=128))
tf.seek(0)
self.assertEqual(digest_full,
generate_md5_hash(tf.name, block_size=256))
tf.seek(0)
self.assertEqual(digest_full,
generate_md5_hash(tf.name, block_size=1024))
tf.seek(0)
self.assertEqual(digest_full,
generate_md5_hash(tf.name,
block_size=100000000)) # 100M
tf.seek(0)
self.assertEqual(digest_full,
generate_md5_hash(tf.name, block_size=-1))
tf.seek(0)
self.assertNotEqual(digest_full,
generate_md5_hash(tf.name, block_size=0))
def test_get_object_datetime(self):
file1_content = "content of the test file1"
self.storage.storage_driver.upload_blob_from_string(
"test_download_blobs1/file1.txt", file1_content)
obj = self.storage.storage_driver.get_blob(
"test_download_blobs1/file1.txt")
self.assertEquals(
datetime.datetime.fromtimestamp(int(obj.extra["modify_time"])),
self.storage.storage_driver.get_object_datetime(obj))
def test_get_fqdn_from_backup_index_blob(self):
blob_name = "index/backup_index/2019051307/manifest_node1.whatever.com.json"
self.assertEquals("node1.whatever.com",
self.storage.get_fqdn_from_any_index_blob(blob_name))
blob_name = "index/backup_index/2019051307/schema_node2.whatever.com.cql"
self.assertEquals("node2.whatever.com",
self.storage.get_fqdn_from_any_index_blob(blob_name))
blob_name = "index/backup_index/2019051307/schema_node3.whatever.com.txt"
self.assertEquals("node3.whatever.com",
self.storage.get_fqdn_from_any_index_blob(blob_name))
blob_name = "index/backup_index/2019051307/schema_node_with_underscores.whatever.com.txt"
self.assertEquals("node_with_underscores.whatever.com",
self.storage.get_fqdn_from_any_index_blob(blob_name))
def test_get_fqdn_from_any_index_blob(self):
blob_name = "tokenmap_hostname-with-dashes-and-3-numbers.json"
self.assertEqual("hostname-with-dashes-and-3-numbers",
self.storage.get_fqdn_from_any_index_blob(blob_name))
blob_name = "tokenmap_hostname-with-dashes.and-dots.json"
self.assertEqual("hostname-with-dashes.and-dots",
self.storage.get_fqdn_from_any_index_blob(blob_name))
blob_name = "tokenmap_hostname_with-underscores.and-dots-and.dashes.json"
self.assertEqual("hostname_with-underscores.and-dots-and.dashes",
self.storage.get_fqdn_from_any_index_blob(blob_name))
blob_name = "index/bi/third_backup/finished_localhost_1574343029.timestamp"
self.assertEqual("localhost",
self.storage.get_fqdn_from_any_index_blob(blob_name))
def test_parse_backup_index(self):
file_content = "content of the test file"
# SSTables for node1 and backup1
self.storage.storage_driver.upload_blob_from_string(
"node1/backup1/data/ks1/sstable1.db", file_content)
self.storage.storage_driver.upload_blob_from_string(
"node1/backup1/data/ks1/sstable2.db", file_content)
# Metadata for node1 and backup1
self.storage.storage_driver.upload_blob_from_string(
"node1/backup1/meta/tokenmap.json", file_content)
self.storage.storage_driver.upload_blob_from_string(
"node1/backup1/meta/manifest.json", file_content)
self.storage.storage_driver.upload_blob_from_string(
"node1/backup1/meta/schema.cql", file_content)
# SSTables for node2 and backup1
self.storage.storage_driver.upload_blob_from_string(
"node2/backup1/data/ks1/sstable1.db", file_content)
self.storage.storage_driver.upload_blob_from_string(
"node2/backup1/data/ks1/sstable2.db", file_content)
# Metadata for node2 and backup1
self.storage.storage_driver.upload_blob_from_string(
"node2/backup1/meta/tokenmap.json", file_content)
self.storage.storage_driver.upload_blob_from_string(
"node2/backup1/meta/manifest.json", file_content)
self.storage.storage_driver.upload_blob_from_string(
"node2/backup1/meta/schema.cql", file_content)
# SSTables for node1 and backup2
self.storage.storage_driver.upload_blob_from_string(
"node1/backup2/data/ks1/sstable1.db", file_content)
self.storage.storage_driver.upload_blob_from_string(
"node1/backup2/data/ks1/sstable2.db", file_content)
# Metadata for node1 and backup2
self.storage.storage_driver.upload_blob_from_string(
"node1/backup2/meta/tokenmap.json", file_content)
self.storage.storage_driver.upload_blob_from_string(
"node1/backup2/meta/manifest.json", file_content)
self.storage.storage_driver.upload_blob_from_string(
"node1/backup2/meta/schema.cql", file_content)
build_indices(self.config, False)
path = 'index/backup_index'
backup_index = self.storage.storage_driver.list_objects(path)
blobs_by_backup = self.storage.group_backup_index_by_backup_and_node(
backup_index)
self.assertTrue("backup1" in blobs_by_backup)
self.assertTrue("backup2" in blobs_by_backup)
self.assertTrue("node1" in blobs_by_backup["backup1"])
self.assertTrue("node2" in blobs_by_backup["backup1"])
self.assertTrue("node1" in blobs_by_backup["backup2"])
self.assertFalse("node2" in blobs_by_backup["backup2"])
def test_remove_extension(self):
self.assertEquals('localhost',
self.storage.remove_extension('localhost.txt'))
self.assertEquals('localhost',
self.storage.remove_extension('localhost.timestamp'))
self.assertEquals('localhost',
self.storage.remove_extension('localhost.cql'))
self.assertEquals('localhost.foo',
self.storage.remove_extension('localhost.foo'))
def test_get_timestamp_from_blob_name(self):
self.assertEquals(
1558021519,
self.storage.get_timestamp_from_blob_name(
'finished_localhost_1558021519.timestamp'))
self.assertEquals(
1558021519,
self.storage.get_timestamp_from_blob_name(
'finished_some.host.net_1558021519.timestamp'))
self.assertEquals(
1558021519,
self.storage.get_timestamp_from_blob_name(
'finished_some_underscores.host.net_1558021519.timestamp'))
self.assertEquals(
1574343029,
self.storage.get_timestamp_from_blob_name(
'index/bi/third_backup/finished_localhost_1574343029.timestamp'
))
def _truncate_the_backup_folder(context):
storage = Storage(config=context.medusa_config.storage)
path_root = "/tmp/medusa_it_bucket"
backup_path = "{}/{}localhost".format(path_root, storage.prefix_path)
shutil.rmtree(backup_path)
def _the_backup_index_does_not_exist(context):
storage = Storage(config=context.medusa_config.storage)
assert False is medusa.index.index_exists(storage)
def _the_latest_complete_cluster_backup_is(context, expected_backup_name):
storage = Storage(config=context.medusa_config.storage)
actual_backup = storage.latest_complete_cluster_backup()
if actual_backup is not None:
assert expected_backup_name == actual_backup.name
def _truncate_the_index(context):
storage = Storage(config=context.medusa_config.storage)
path_root = "/tmp/medusa_it_bucket"
index_path = "{}/{}index".format(path_root, storage.prefix_path)
shutil.rmtree(index_path)
def _the_latest_cluster_backup_is(context, expected_backup_name):
storage = Storage(config=context.medusa_config.storage)
backup = storage.latest_cluster_backup()
assert expected_backup_name == backup.name
def orchestrate(config, backup_name, stagger, enable_md5_checks, mode,
temp_dir, parallel_snapshots, parallel_uploads):
backup = None
monitoring = Monitoring(config=config.monitoring)
try:
backup_start_time = datetime.datetime.now()
if not config.storage.fqdn:
err_msg = "The fqdn was not provided nor calculated properly."
logging.error(err_msg)
raise Exception(err_msg)
if not temp_dir.is_dir():
err_msg = '{} is not a directory'.format(temp_dir)
logging.error(err_msg)
raise Exception(err_msg)
try:
# Try to get a backup with backup_name. If it exists then we cannot take another backup with that name
storage = Storage(config=config.storage)
cluster_backup = storage.get_cluster_backup(backup_name)
if cluster_backup:
err_msg = 'Backup named {} already exists.'.format(backup_name)
logging.error(err_msg)
raise Exception(err_msg)
except KeyError:
info_msg = 'Starting backup {}'.format(backup_name)
logging.info(info_msg)
backup = BackupJob(config, backup_name, stagger, enable_md5_checks,
mode, temp_dir, parallel_snapshots,
parallel_uploads)
backup.execute()
backup_end_time = datetime.datetime.now()
backup_duration = backup_end_time - backup_start_time
logging.debug('Emitting metrics')
logging.info('Backup duration: {}'.format(backup_duration.seconds))
tags = [
'medusa-cluster-backup', 'cluster-backup-duration', backup_name
]
monitoring.send(tags, backup_duration.seconds)
tags = ['medusa-cluster-backup', 'cluster-backup-error', backup_name]
monitoring.send(tags, 0)
logging.debug('Done emitting metrics.')
logging.info('Backup of the cluster done.')
except Exception as e:
tags = ['medusa-cluster-backup', 'cluster-backup-error', backup_name]
monitoring.send(tags, 1)
logging.error(
'This error happened during the cluster backup: {}'.format(str(e)))
traceback.print_exc()
if backup is not None:
err_msg = 'Something went wrong! Attempting to clean snapshots and exit.'
logging.error(err_msg)
delete_snapshot_command = ' '.join(
backup.cassandra.delete_snapshot_command(backup.snapshot_tag))
pssh_run_success_cleanup = backup.orchestration_uploads\
.pssh_run(backup.hosts,
delete_snapshot_command,
hosts_variables={})
if pssh_run_success_cleanup:
info_msg = 'All nodes successfully cleared their snapshot.'
logging.info(info_msg)
else:
err_msg_cleanup = 'Some nodes failed to clear the snapshot. Cleaning snapshots manually is recommended'
logging.error(err_msg_cleanup)
sys.exit(1)
def _there_is_no_latest_backup_for_node_fqdn(context, fqdn):
storage = Storage(config=context.medusa_config.storage)
node_backup = storage.latest_node_backup(fqdn=fqdn)
assert node_backup is None
def _node_fakes_a_complete_backup(context, fqdn, backup_name, backup_datetime):
storage = Storage(config=context.medusa_config.storage)
path_root = "/tmp/medusa_it_bucket"
fake_tokenmap = json.dumps({
"n1": {
"tokens": [1],
"is_up": True
},
"n2": {
"tokens": [2],
"is_up": True
},
"n3": {
"tokens": [3],
"is_up": True
},
})
dir_path = os.path.join(path_root, storage.prefix_path + "index",
"backup_index", backup_name)
if not os.path.exists(dir_path):
os.makedirs(dir_path)
# fake token map, manifest and schema in index
path_tokenmap = "{}/{}index/backup_index/{}/tokenmap_{}.json".format(
path_root, storage.prefix_path, backup_name, fqdn)
write_dummy_file(path_tokenmap, backup_datetime, fake_tokenmap)
path_manifest = "{}/{}index/backup_index/{}/manifest_{}.json".format(
path_root, storage.prefix_path, backup_name, fqdn)
write_dummy_file(path_manifest, backup_datetime, fake_tokenmap)
path_schema = "{}/{}index/backup_index/{}/schema_{}.cql".format(
path_root, storage.prefix_path, backup_name, fqdn)
write_dummy_file(path_schema, backup_datetime, fake_tokenmap)
dir_path = os.path.join(path_root, storage.prefix_path + "index",
"latest_backup", fqdn)
if not os.path.exists(dir_path):
os.makedirs(dir_path)
# fake token map in latest_backup
path_latest_backup_tokenmap = "{}/{}index/latest_backup/{}/tokenmap.json".format(
path_root, storage.prefix_path, fqdn)
write_dummy_file(path_latest_backup_tokenmap, backup_datetime,
fake_tokenmap)
# fake token name in latest_backup
path_latest_backup_name = "{}/{}index/latest_backup/{}/backup_name.txt".format(
path_root, storage.prefix_path, fqdn)
write_dummy_file(path_latest_backup_name, backup_datetime)
# fake actual backup folder
dir_path = os.path.join(path_root, storage.prefix_path + fqdn, backup_name,
"meta")
if not os.path.exists(dir_path):
os.makedirs(dir_path)
# fake schema in actual backup path
path_schema = "{}/{}{}/{}/meta/schema.cql".format(path_root,
storage.prefix_path,
fqdn, backup_name)
write_dummy_file(path_schema, backup_datetime)
# fake manifest in actual backup path
path_manifest = "{}/{}{}/{}/meta/manifest.json".format(
path_root, storage.prefix_path, fqdn, backup_name)
write_dummy_file(path_manifest, backup_datetime)
# fake token map in actual backup path
path_tokenmap = "{}/{}{}/{}/meta/tokenmap.json".format(
path_root, storage.prefix_path, fqdn, backup_name)
write_dummy_file(path_tokenmap, backup_datetime, fake_tokenmap)
def main(config, backup_name_arg, stagger_time, mode):
start = datetime.datetime.now()
backup_name = backup_name_arg or start.strftime('%Y%m%d%H')
monitoring = Monitoring(config=config.monitoring)
try:
storage = Storage(config=config.storage)
cassandra = Cassandra(config)
differential_mode = False
if mode == "differential":
differential_mode = True
node_backup = storage.get_node_backup(
fqdn=config.storage.fqdn,
name=backup_name,
differential_mode=differential_mode)
if node_backup.exists():
raise IOError(
'Error: Backup {} already exists'.format(backup_name))
# Make sure that priority remains to Cassandra/limiting backups resource usage
try:
throttle_backup()
except Exception:
logging.warning(
"Throttling backup impossible. It's probable that ionice is not available."
)
logging.info('Saving tokenmap and schema')
schema, tokenmap = get_schema_and_tokenmap(cassandra)
node_backup.schema = schema
node_backup.tokenmap = json.dumps(tokenmap)
if differential_mode is True:
node_backup.differential = mode
add_backup_start_to_index(storage, node_backup)
if stagger_time:
stagger_end = start + stagger_time
logging.info(
'Staggering backup run, trying until {}'.format(stagger_end))
while not stagger(config.storage.fqdn, storage, tokenmap):
if datetime.datetime.now() < stagger_end:
logging.info('Staggering this backup run...')
time.sleep(60)
else:
raise IOError('Backups on previous nodes did not complete'
' within our stagger time.')
actual_start = datetime.datetime.now()
num_files, node_backup_cache = do_backup(cassandra, node_backup,
storage, differential_mode,
config, backup_name)
end = datetime.datetime.now()
actual_backup_duration = end - actual_start
print_backup_stats(actual_backup_duration, actual_start, end,
node_backup, node_backup_cache, num_files, start)
update_monitoring(actual_backup_duration, backup_name, monitoring,
node_backup)
return (actual_backup_duration, actual_start, end, node_backup,
node_backup_cache, num_files, start)
except Exception as e:
tags = ['medusa-node-backup', 'backup-error', backup_name]
monitoring.send(tags, 1)
medusa.utils.handle_exception(
e, "This error happened during the backup: {}".format(str(e)),
config)
def __init__(self, config):
logging.info("Init service")
self.config = config
self.storage = Storage(config=self.config.storage)
class MedusaService(medusa_pb2_grpc.MedusaServicer):
def __init__(self, config):
logging.info("Init service")
self.config = config
self.storage = Storage(config=self.config.storage)
def AsyncBackup(self, request, context):
# TODO pass the staggered arg
logging.info("Performing ASYNC backup {} (type={})".format(
request.name, request.mode))
response = medusa_pb2.BackupResponse()
mode = BACKUP_MODE_DIFFERENTIAL
if medusa_pb2.BackupRequest.Mode.FULL == request.mode:
mode = BACKUP_MODE_FULL
try:
response.backupName = request.name
response.status = response.status = medusa_pb2.StatusType.IN_PROGRESS
with ThreadPoolExecutor(
max_workers=1,
thread_name_prefix=request.name) as executor:
BackupMan.register_backup(request.name, is_async=True)
backup_future = executor.submit(backup_node.handle_backup,
config=self.config,
backup_name_arg=request.name,
stagger_time=None,
enable_md5_checks_flag=False,
mode=mode)
backup_future.add_done_callback(record_backup_info)
BackupMan.set_backup_future(request.name, backup_future)
except Exception as e:
response.status = medusa_pb2.StatusType.FAILED
if request.name:
BackupMan.update_backup_status(request.name,
BackupMan.STATUS_FAILED)
context.set_details("Failed to create async backup: {}".format(e))
context.set_code(grpc.StatusCode.INTERNAL)
logging.exception("Async backup failed due to error: {}".format(e))
return response
def Backup(self, request, context):
# TODO pass the staggered arg
logging.info("Performing SYNC backup {} (type={})".format(
request.name, request.mode))
response = medusa_pb2.BackupResponse()
mode = BACKUP_MODE_DIFFERENTIAL
if medusa_pb2.BackupRequest.Mode.FULL == request.mode:
mode = BACKUP_MODE_FULL
try:
response.backupName = request.name
BackupMan.register_backup(request.name, is_async=False)
backup_node.handle_backup(config=self.config,
backup_name_arg=request.name,
stagger_time=None,
enable_md5_checks_flag=False,
mode=mode)
record_status_in_response(response, request.name)
return response
except Exception as e:
response.status = medusa_pb2.StatusType.FAILED
if request.name:
BackupMan.update_backup_status(request.name,
BackupMan.STATUS_FAILED)
context.set_details("Failed to create sync backups: {}".format(e))
context.set_code(grpc.StatusCode.INTERNAL)
logging.exception("Sync backup failed due to error: {}".format(e))
return response
def BackupStatus(self, request, context):
response = medusa_pb2.BackupStatusResponse()
try:
backup = self.storage.get_cluster_backup(request.backupName)
# TODO how is the startTime determined?
response.startTime = datetime.fromtimestamp(
backup.started).strftime(TIMESTAMP_FORMAT)
response.finishedNodes.extend(
[node.fqdn for node in backup.complete_nodes()])
response.unfinishedNodes.extend(
[node.fqdn for node in backup.incomplete_nodes()])
response.missingNodes.extend(
[node.fqdn for node in backup.missing_nodes()])
if backup.finished:
response.finishTime = datetime.fromtimestamp(
backup.finished).strftime(TIMESTAMP_FORMAT)
else:
response.finishTime = ""
record_status_in_response(response, request.backupName)
except KeyError:
context.set_details("backup <{}> does not exist".format(
request.backupName))
context.set_code(grpc.StatusCode.NOT_FOUND)
response.status = medusa_pb2.StatusType.UNKNOWN
return response
def GetBackups(self, request, context):
response = medusa_pb2.GetBackupsResponse()
last_status = medusa_pb2.StatusType.UNKNOWN
try:
# cluster backups
backups = get_backups(self.config, True)
for backup in backups:
summary = medusa_pb2.BackupSummary()
summary.backupName = backup.name
if backup.started is None:
summary.startTime = 0
else:
summary.startTime = backup.started
if backup.finished is None:
summary.finishTime = 0
summary.status = medusa_pb2.StatusType.IN_PROGRESS
last_status = medusa_pb2.StatusType.IN_PROGRESS
else:
summary.finishTime = backup.finished
if last_status != medusa_pb2.StatusType.IN_PROGRESS:
summary.status = medusa_pb2.StatusType.SUCCESS
summary.totalNodes = len(backup.tokenmap)
summary.finishedNodes = len(backup.complete_nodes())
for node in backup.tokenmap:
summary.nodes.append(create_token_map_node(backup, node))
response.backups.append(summary)
except Exception as e:
context.set_details(
"Failed to get backups due to error: {}".format(e))
context.set_code(grpc.StatusCode.INTERNAL)
response.status = medusa_pb2.StatusType.UNKNOWN
return response
def DeleteBackup(self, request, context):
logging.info("Deleting backup {}".format(request.name))
response = medusa_pb2.DeleteBackupResponse()
try:
delete_backup(self.config, [request.name], True)
handle_backup_removal(request.name)
except Exception as e:
context.set_details("deleting backups failed: {}".format(e))
context.set_code(grpc.StatusCode.INTERNAL)
logging.exception("Deleting backup {} failed".format(request.name))
return response
def _i_can_see_no_backups(context):
storage = Storage(config=context.medusa_config.storage)
cluster_backups = storage.list_cluster_backups()
assert 0 == len(list(cluster_backups))
def _there_is_no_latest_complete_backup(context):
storage = Storage(config=context.medusa_config.storage)
actual_backup = storage.latest_complete_cluster_backup()
assert actual_backup is None
def _the_latest_backup_for_fqdn_is_called_backupname(context, expected_fqdn,
expected_backup_name):
storage = Storage(config=context.medusa_config.storage)
latest_backup = storage.latest_node_backup(fqdn=expected_fqdn)
assert latest_backup.name == expected_backup_name
def orchestrate(config, backup_name, seed_target, temp_dir, host_list, keep_auth, bypass_checks,
verify, keyspaces, tables, pssh_pool_size, use_sstableloader=False):
monitoring = Monitoring(config=config.monitoring)
try:
restore_start_time = datetime.datetime.now()
if seed_target is not None:
keep_auth = False
if seed_target is None and host_list is None:
err_msg = 'You must either provide a seed target or a list of host'
logging.error(err_msg)
raise Exception(err_msg)
if seed_target is not None and host_list is not None:
err_msg = 'You must either provide a seed target or a list of host, not both'
logging.error(err_msg)
raise Exception(err_msg)
if not temp_dir.is_dir():
err_msg = '{} is not a directory'.format(temp_dir)
logging.error(err_msg)
raise Exception(err_msg)
if keep_auth:
logging.info('system_auth keyspace will be left untouched on the target nodes')
else:
logging.info('system_auth keyspace will be overwritten with the backup on target nodes')
storage = Storage(config=config.storage)
try:
cluster_backup = storage.get_cluster_backup(backup_name)
except KeyError:
err_msg = 'No such backup --> {}'.format(backup_name)
logging.error(err_msg)
raise Exception(err_msg)
restore = RestoreJob(cluster_backup, config, temp_dir, host_list, seed_target, keep_auth, verify,
pssh_pool_size, keyspaces, tables, bypass_checks, use_sstableloader)
restore.execute()
restore_end_time = datetime.datetime.now()
restore_duration = restore_end_time - restore_start_time
logging.debug('Emitting metrics')
logging.info('Restore duration: {}'.format(restore_duration.seconds))
tags = ['medusa-cluster-restore', 'restore-duration', backup_name]
monitoring.send(tags, restore_duration.seconds)
tags = ['medusa-cluster-restore', 'restore-error', backup_name]
monitoring.send(tags, 0)
logging.debug('Done emitting metrics')
logging.info('Successfully restored the cluster')
except Exception as e:
tags = ['medusa-cluster-restore', 'restore-error', backup_name]
monitoring.send(tags, 1)
logging.error('This error happened during the cluster restore: {}'.format(str(e)))
traceback.print_exc()
sys.exit(1)
