def test_download_from_bucket(self):
# Suppress logging output.
flexmock(logging).should_receive('error').and_return()
# Test with invalid GCS object name.
flexmock(gcs_helper).should_receive('extract_gcs_tokens').\
and_return(('', ''))
self.assertEquals(
False,
gcs_helper.download_from_bucket(FakeInvalidGCSPath, 'some/file'))
# Test with valid GCS object name.
flexmock(gcs_helper).should_receive(
'extract_gcs_tokens').with_args(FakeGCSPath).\
and_return(('foo', 'bar/baz.tar.gz'))
flexmock(urllib).should_receive('quote_plus').and_return('')
# Test with unsuccessful GET request.
response = flexmock(status_code=404)
flexmock(gcs_helper).should_receive('gcs_get_request').\
and_return(response)
self.assertEquals(
False,
gcs_helper.download_from_bucket(FakeInvalidGCSPath, 'some/file'))
# Test with HTTPError from GET request.
flexmock(gcs_helper).should_receive('gcs_get_request').\
and_raise(requests.HTTPError)
self.assertEquals(
False, gcs_helper.download_from_bucket(FakeGCSPath, 'some/file'))
# Test with successful GET request.
response = flexmock(status_code=200,
content=json.dumps({
'size': 1024,
'mediaLink': 'some/link'
}))
flexmock(gcs_helper).should_receive('gcs_get_request').\
and_return(response)
# Test with insufficient disk space.
disk_stats = flexmock(f_bavail=0, f_frsize=4096)
flexmock(os).should_receive('statvfs').and_return(disk_stats)
self.assertEquals(
False, gcs_helper.download_from_bucket(FakeGCSPath, 'some/file'))
# Test with sufficient disk space.
disk_stats = flexmock(f_bavail=1024, f_frsize=4096)
flexmock(os).should_receive('statvfs').and_return(disk_stats)
flexmock(subprocess).should_receive('check_output').and_return()
self.assertEquals(
True, gcs_helper.download_from_bucket(FakeGCSPath, 'some/file'))
def test_download_from_bucket(self):
# Suppress logging output.
flexmock(logging).should_receive('error').and_return()
# Test with invalid GCS object name.
flexmock(gcs_helper).should_receive('extract_gcs_tokens').\
and_return(('', ''))
self.assertEquals(False, gcs_helper.download_from_bucket(FakeInvalidGCSPath,
'some/file'))
# Test with valid GCS object name.
flexmock(gcs_helper).should_receive(
'extract_gcs_tokens').with_args(FakeGCSPath).\
and_return(('foo', 'bar/baz.tar.gz'))
flexmock(urllib).should_receive('quote_plus').and_return('')
# Test with unsuccessful GET request.
response = flexmock(status_code=404)
flexmock(gcs_helper).should_receive('gcs_get_request').\
and_return(response)
self.assertEquals(False, gcs_helper.download_from_bucket(FakeInvalidGCSPath,
'some/file'))
# Test with HTTPError from GET request.
flexmock(gcs_helper).should_receive('gcs_get_request').\
and_raise(requests.HTTPError)
self.assertEquals(False, gcs_helper.download_from_bucket(FakeGCSPath,
'some/file'))
# Test with successful GET request.
response = flexmock(status_code=200,
content=json.dumps({'size': 1024, 'mediaLink': 'some/link'}))
flexmock(gcs_helper).should_receive('gcs_get_request').\
and_return(response)
# Test with insufficient disk space.
disk_stats = flexmock(f_bavail=0, f_frsize=4096)
flexmock(os).should_receive('statvfs').and_return(disk_stats)
self.assertEquals(False, gcs_helper.download_from_bucket(FakeGCSPath,
'some/file'))
# Test with sufficient disk space.
disk_stats = flexmock(f_bavail=1024, f_frsize=4096)
flexmock(os).should_receive('statvfs').and_return(disk_stats)
flexmock(subprocess).should_receive('check_output').and_return()
self.assertEquals(True, gcs_helper.download_from_bucket(FakeGCSPath,
'some/file'))
def restore_data(storage, path=''):
""" Restores the Cassandra backup.
Args:
storage: A str, one of the StorageTypes class members.
path: A str, the name of the backup file to restore from.
Returns:
True on success, False otherwise.
"""
if storage not in StorageTypes().get_storage_types():
logging.error("Storage '{0}' not supported.")
return False
logging.info("Starting new db restore.")
if storage == StorageTypes.GCS:
# Download backup file and store locally with a fixed name.
if not gcs_helper.download_from_bucket(path,
CASSANDRA_BACKUP_FILE_LOCATION):
logging.error("Download from GCS failed. Aborting recovery...")
return False
# TODO Make sure there's a snapshot to rollback to if restore fails.
# Not pressing for fresh deployments.
# create_snapshot('rollback-snapshot')
if not shut_down_cassandra.run():
logging.error("Unable to shut down Cassandra. Aborting restore...")
if storage == StorageTypes.GCS:
backup_recovery_helper.\
delete_local_backup_file(CASSANDRA_BACKUP_FILE_LOCATION)
return False
remove_old_data()
try:
backup_recovery_helper.untar_backup_files(CASSANDRA_BACKUP_FILE_LOCATION)
except backup_exceptions.BRException as br_exception:
logging.exception("Error while unpacking backup files. Exception: {0}".
format(str(br_exception)))
start_cassandra.run()
if storage == StorageTypes.GCS:
backup_recovery_helper.\
delete_local_backup_file(CASSANDRA_BACKUP_FILE_LOCATION)
return False
restore_snapshots()
# Start Cassandra and repair.
logging.info("Starting Cassandra.")
start_cassandra.run()
refresh_data()
# Local cleanup.
clear_old_snapshots()
if storage == StorageTypes.GCS:
backup_recovery_helper.\
delete_local_backup_file(CASSANDRA_BACKUP_FILE_LOCATION)
logging.info("Done with db restore.")
return True
def test_download_from_bucket(self):
# Suppress logging output.
flexmock(logging).should_receive('error').and_return()
# Test with invalid GCS object name.
flexmock(gcs_helper).should_receive('extract_gcs_tokens').\
and_return(('', ''))
self.assertEquals(False, gcs_helper.download_from_bucket(FakeInvalidGCSPath,
'some/file'))
# Test with valid GCS object name.
flexmock(gcs_helper).should_receive(
'extract_gcs_tokens').with_args(FakeGCSPath).\
and_return(('foo', 'bar/baz.tar.gz'))
flexmock(urllib).should_receive('quote_plus').and_return('')
# Test with unsuccessful GET request.
flexmock(gcs_helper).should_receive('gcs_get_request').\
and_return(FakeResponse2(0))
self.assertEquals(False, gcs_helper.download_from_bucket(FakeInvalidGCSPath,
'some/file'))
# Test with HTTPError from GET request.
flexmock(gcs_helper).should_receive('gcs_get_request').\
and_raise(requests.HTTPError)
self.assertEquals(False, gcs_helper.download_from_bucket(FakeGCSPath,
'some/file'))
# Test with successful GET request.
flexmock(gcs_helper).should_receive('gcs_get_request').\
and_return(FakeResponse2(200))
# Test with insufficient disk space.
flexmock(json).should_receive('loads').and_return({"size": "2"})
flexmock(subprocess).should_receive('check_output').and_return(
FakeDFOutput)
flexmock(re).should_receive('sub').and_return(FakeFilteredDFOutput)
self.assertEquals(False, gcs_helper.download_from_bucket(FakeGCSPath,
'some/file'))
# Test with sufficient disk space.
flexmock(json).should_receive('loads').and_return({
"size": "0", "mediaLink": "some/link"})
flexmock(subprocess).should_receive('check_output').and_return(
FakeDFOutput)
flexmock(re).should_receive('sub').and_return(FakeFilteredDFOutput)
self.assertEquals(True, gcs_helper.download_from_bucket(FakeGCSPath,
'some/file'))
def restore_data(storage, path=''):
""" Restores the Zookeeper snapshot.
Args:
storage: A str, one of the StorageTypes class members.
path: A str, the name of the backup file to restore from.
"""
if storage not in StorageTypes().get_storage_types():
logging.error("Storage '{0}' not supported.")
return False
logging.info("Starting new zk restore.")
if storage == StorageTypes.GCS:
# Download backup file and store locally with a fixed name.
if not gcs_helper.download_from_bucket(path,
ZOOKEEPER_BACKUP_FILE_LOCATION):
logging.error("Download from GCS failed. Aborting recovery...")
return False
# TODO Make sure there's a snapshot to rollback to if restore fails.
# Not pressing for fresh deployments.
flush_zk()
try:
backup_recovery_helper.untar_backup_files(
ZOOKEEPER_BACKUP_FILE_LOCATION)
except backup_exceptions.BRException as br_exception:
logging.exception(
"Error while unpacking backup files. Exception: {0}".format(
str(br_exception)))
if storage == StorageTypes.GCS:
backup_recovery_helper.\
delete_local_backup_file(ZOOKEEPER_BACKUP_FILE_LOCATION)
return False
restore_zk(TMP_ZOOKEEPER_BACKUP)
# Local cleanup.
backup_recovery_helper.remove(TMP_ZOOKEEPER_BACKUP)
if storage == StorageTypes.GCS:
backup_recovery_helper.\
delete_local_backup_file(ZOOKEEPER_BACKUP_FILE_LOCATION)
logging.info("Done with zk restore.")
return True
def app_restore(storage, bucket_name=None):
""" Restores the app source code from the backups location on the filesystem.
Args:
storage: A str, one of the StorageTypes class members.
bucket_name: A str, the name of the bucket to restore apps from.
Returns:
True on success, False otherwise.
"""
# Create app backups dir if it doesn't exist.
if not makedirs(APP_BACKUP_DIR_LOCATION):
logging.warning(
"Dir '{0}' already exists. Skipping dir creation...".format(
APP_BACKUP_DIR_LOCATION))
# Download from GCS to backups location.
if storage == StorageTypes.GCS:
objects = gcs_helper.list_bucket(bucket_name)
for app_path in objects:
if not app_path.startswith(gcs_helper.APPS_GCS_PREFIX):
continue
# Only keep the relative name of the app file.
# E.g. myapp.tar.gz (app_file) out of apps/myapp.tar.gz (app_path)
app_file = app_path[len(gcs_helper.APPS_GCS_PREFIX):]
source = 'gs://{0}/{1}'.format(bucket_name, app_path)
destination = '{0}/{1}'.format(APP_BACKUP_DIR_LOCATION, app_file)
if not gcs_helper.download_from_bucket(source, destination):
logging.error(
"Error while downloading '{0}' from GCS.".format(source))
delete_app_tars(APP_BACKUP_DIR_LOCATION)
return False
# Deploy apps.
apps_to_deploy = [
os.path.join(APP_BACKUP_DIR_LOCATION, app)
for app in os.listdir(APP_BACKUP_DIR_LOCATION)
]
if not deploy_apps(apps_to_deploy):
logging.error("Failed to successfully deploy one or more of the "
"following apps: {0}".format(apps_to_deploy))
return False
return True
def restore_data(storage, path=''):
""" Restores the Zookeeper snapshot.
Args:
storage: A str, one of the StorageTypes class members.
path: A str, the name of the backup file to restore from.
"""
if storage not in StorageTypes().get_storage_types():
logging.error("Storage '{0}' not supported.")
return False
logging.info("Starting new zk restore.")
if storage == StorageTypes.GCS:
# Download backup file and store locally with a fixed name.
if not gcs_helper.download_from_bucket(path,
ZOOKEEPER_BACKUP_FILE_LOCATION):
logging.error("Download from GCS failed. Aborting recovery...")
return False
# TODO Make sure there's a snapshot to rollback to if restore fails.
# Not pressing for fresh deployments.
flush_zk()
try:
backup_recovery_helper.untar_backup_files(ZOOKEEPER_BACKUP_FILE_LOCATION)
except backup_exceptions.BRException as br_exception:
logging.exception("Error while unpacking backup files. Exception: {0}".
format(str(br_exception)))
if storage == StorageTypes.GCS:
backup_recovery_helper.\
delete_local_backup_file(ZOOKEEPER_BACKUP_FILE_LOCATION)
return False
restore_zk(TMP_ZOOKEEPER_BACKUP)
# Local cleanup.
backup_recovery_helper.remove(TMP_ZOOKEEPER_BACKUP)
if storage == StorageTypes.GCS:
backup_recovery_helper.\
delete_local_backup_file(ZOOKEEPER_BACKUP_FILE_LOCATION)
logging.info("Done with zk restore.")
return True
def app_restore(storage, bucket_name=None):
""" Restores the app source code from the backups location on the filesystem.
Args:
storage: A str, one of the StorageTypes class members.
bucket_name: A str, the name of the bucket to restore apps from.
Returns:
True on success, False otherwise.
"""
# Create app backups dir if it doesn't exist.
if not makedirs(APP_BACKUP_DIR_LOCATION):
logging.warning("Dir '{0}' already exists. Skipping dir creation...".
format(APP_BACKUP_DIR_LOCATION))
# Download from GCS to backups location.
if storage == StorageTypes.GCS:
objects = gcs_helper.list_bucket(bucket_name)
for app_path in objects:
if not app_path.startswith(gcs_helper.APPS_GCS_PREFIX):
continue
# Only keep the relative name of the app file.
# E.g. myapp.tar.gz (app_file) out of apps/myapp.tar.gz (app_path)
app_file = app_path[len(gcs_helper.APPS_GCS_PREFIX):]
source = 'gs://{0}/{1}'.format(bucket_name, app_path)
destination = '{0}/{1}'.format(APP_BACKUP_DIR_LOCATION, app_file)
if not gcs_helper.download_from_bucket(source, destination):
logging.error("Error while downloading '{0}' from GCS.".format(source))
delete_app_tars(APP_BACKUP_DIR_LOCATION)
return False
# Deploy apps.
apps_to_deploy = [os.path.join(APP_BACKUP_DIR_LOCATION, app) for app in
os.listdir(APP_BACKUP_DIR_LOCATION)]
if not deploy_apps(apps_to_deploy):
logging.error("Failed to successfully deploy one or more of the "
"following apps: {0}".format(apps_to_deploy))
return False
return True
def restore_data(storage, path=''):
""" Restores the Zookeeper snapshot.
Args:
storage: A str, one of the StorageTypes class members.
path: A str, the name of the backup file to restore from.
"""
if storage == StorageTypes.GCS:
# Download backup file and store locally with a fixed name.
if not gcs_helper.download_from_bucket(path, BACKUP_FILE_LOCATION):
logging.error("Download from GCS failed. Aborting recovery...")
return False
# TODO Make sure there's a snapshot to rollback to if restore fails.
# Not pressing for fresh deployments.
if not shut_down_zookeeper.run():
logging.error("Unable to shut down Zookeeper. Aborting restore...")
return False
remove_old_data()
try:
untar_backup_files()
except backup_exceptions.BRException as br_exception:
logging.exception("Exception while restoring zk snapshots. Need to "
"rollback... Exception: {0}".format(
str(br_exception)))
restore_snapshots()
# Start Zookeeper.
start_zookeeper.run()
# Local cleanup.
if storage == StorageTypes.GCS:
remove_local_backup_file()
logging.info("Done with restore.")
return True
def restore_data(storage, path=''):
""" Restores the Zookeeper snapshot.
Args:
storage: A str, one of the StorageTypes class members.
path: A str, the name of the backup file to restore from.
"""
if storage == StorageTypes.GCS:
# Download backup file and store locally with a fixed name.
if not gcs_helper.download_from_bucket(path, BACKUP_FILE_LOCATION):
logging.error("Download from GCS failed. Aborting recovery...")
return False
# TODO Make sure there's a snapshot to rollback to if restore fails.
# Not pressing for fresh deployments.
if not shut_down_zookeeper.run():
logging.error("Unable to shut down Zookeeper. Aborting restore...")
return False
remove_old_data()
try:
untar_backup_files()
except backup_exceptions.BRException as br_exception:
logging.exception("Exception while restoring zk snapshots. Need to "
"rollback... Exception: {0}".format(str(br_exception)))
restore_snapshots()
# Start Zookeeper.
start_zookeeper.run()
# Local cleanup.
if storage == StorageTypes.GCS:
remove_local_backup_file()
logging.info("Done with restore.")
return True
