def test_download_from_bucket(self): # Suppress logging output. flexmock(logging).should_receive('error').and_return() # Test with invalid GCS object name. flexmock(gcs_helper).should_receive('extract_gcs_tokens').\ and_return(('', '')) self.assertEquals( False, gcs_helper.download_from_bucket(FakeInvalidGCSPath, 'some/file')) # Test with valid GCS object name. flexmock(gcs_helper).should_receive( 'extract_gcs_tokens').with_args(FakeGCSPath).\ and_return(('foo', 'bar/baz.tar.gz')) flexmock(urllib).should_receive('quote_plus').and_return('') # Test with unsuccessful GET request. response = flexmock(status_code=404) flexmock(gcs_helper).should_receive('gcs_get_request').\ and_return(response) self.assertEquals( False, gcs_helper.download_from_bucket(FakeInvalidGCSPath, 'some/file')) # Test with HTTPError from GET request. flexmock(gcs_helper).should_receive('gcs_get_request').\ and_raise(requests.HTTPError) self.assertEquals( False, gcs_helper.download_from_bucket(FakeGCSPath, 'some/file')) # Test with successful GET request. response = flexmock(status_code=200, content=json.dumps({ 'size': 1024, 'mediaLink': 'some/link' })) flexmock(gcs_helper).should_receive('gcs_get_request').\ and_return(response) # Test with insufficient disk space. disk_stats = flexmock(f_bavail=0, f_frsize=4096) flexmock(os).should_receive('statvfs').and_return(disk_stats) self.assertEquals( False, gcs_helper.download_from_bucket(FakeGCSPath, 'some/file')) # Test with sufficient disk space. disk_stats = flexmock(f_bavail=1024, f_frsize=4096) flexmock(os).should_receive('statvfs').and_return(disk_stats) flexmock(subprocess).should_receive('check_output').and_return() self.assertEquals( True, gcs_helper.download_from_bucket(FakeGCSPath, 'some/file'))
def test_download_from_bucket(self): # Suppress logging output. flexmock(logging).should_receive('error').and_return() # Test with invalid GCS object name. flexmock(gcs_helper).should_receive('extract_gcs_tokens').\ and_return(('', '')) self.assertEquals(False, gcs_helper.download_from_bucket(FakeInvalidGCSPath, 'some/file')) # Test with valid GCS object name. flexmock(gcs_helper).should_receive( 'extract_gcs_tokens').with_args(FakeGCSPath).\ and_return(('foo', 'bar/baz.tar.gz')) flexmock(urllib).should_receive('quote_plus').and_return('') # Test with unsuccessful GET request. response = flexmock(status_code=404) flexmock(gcs_helper).should_receive('gcs_get_request').\ and_return(response) self.assertEquals(False, gcs_helper.download_from_bucket(FakeInvalidGCSPath, 'some/file')) # Test with HTTPError from GET request. flexmock(gcs_helper).should_receive('gcs_get_request').\ and_raise(requests.HTTPError) self.assertEquals(False, gcs_helper.download_from_bucket(FakeGCSPath, 'some/file')) # Test with successful GET request. response = flexmock(status_code=200, content=json.dumps({'size': 1024, 'mediaLink': 'some/link'})) flexmock(gcs_helper).should_receive('gcs_get_request').\ and_return(response) # Test with insufficient disk space. disk_stats = flexmock(f_bavail=0, f_frsize=4096) flexmock(os).should_receive('statvfs').and_return(disk_stats) self.assertEquals(False, gcs_helper.download_from_bucket(FakeGCSPath, 'some/file')) # Test with sufficient disk space. disk_stats = flexmock(f_bavail=1024, f_frsize=4096) flexmock(os).should_receive('statvfs').and_return(disk_stats) flexmock(subprocess).should_receive('check_output').and_return() self.assertEquals(True, gcs_helper.download_from_bucket(FakeGCSPath, 'some/file'))
def restore_data(storage, path=''): """ Restores the Cassandra backup. Args: storage: A str, one of the StorageTypes class members. path: A str, the name of the backup file to restore from. Returns: True on success, False otherwise. """ if storage not in StorageTypes().get_storage_types(): logging.error("Storage '{0}' not supported.") return False logging.info("Starting new db restore.") if storage == StorageTypes.GCS: # Download backup file and store locally with a fixed name. if not gcs_helper.download_from_bucket(path, CASSANDRA_BACKUP_FILE_LOCATION): logging.error("Download from GCS failed. Aborting recovery...") return False # TODO Make sure there's a snapshot to rollback to if restore fails. # Not pressing for fresh deployments. # create_snapshot('rollback-snapshot') if not shut_down_cassandra.run(): logging.error("Unable to shut down Cassandra. Aborting restore...") if storage == StorageTypes.GCS: backup_recovery_helper.\ delete_local_backup_file(CASSANDRA_BACKUP_FILE_LOCATION) return False remove_old_data() try: backup_recovery_helper.untar_backup_files(CASSANDRA_BACKUP_FILE_LOCATION) except backup_exceptions.BRException as br_exception: logging.exception("Error while unpacking backup files. Exception: {0}". format(str(br_exception))) start_cassandra.run() if storage == StorageTypes.GCS: backup_recovery_helper.\ delete_local_backup_file(CASSANDRA_BACKUP_FILE_LOCATION) return False restore_snapshots() # Start Cassandra and repair. logging.info("Starting Cassandra.") start_cassandra.run() refresh_data() # Local cleanup. clear_old_snapshots() if storage == StorageTypes.GCS: backup_recovery_helper.\ delete_local_backup_file(CASSANDRA_BACKUP_FILE_LOCATION) logging.info("Done with db restore.") return True
def test_download_from_bucket(self): # Suppress logging output. flexmock(logging).should_receive('error').and_return() # Test with invalid GCS object name. flexmock(gcs_helper).should_receive('extract_gcs_tokens').\ and_return(('', '')) self.assertEquals(False, gcs_helper.download_from_bucket(FakeInvalidGCSPath, 'some/file')) # Test with valid GCS object name. flexmock(gcs_helper).should_receive( 'extract_gcs_tokens').with_args(FakeGCSPath).\ and_return(('foo', 'bar/baz.tar.gz')) flexmock(urllib).should_receive('quote_plus').and_return('') # Test with unsuccessful GET request. flexmock(gcs_helper).should_receive('gcs_get_request').\ and_return(FakeResponse2(0)) self.assertEquals(False, gcs_helper.download_from_bucket(FakeInvalidGCSPath, 'some/file')) # Test with HTTPError from GET request. flexmock(gcs_helper).should_receive('gcs_get_request').\ and_raise(requests.HTTPError) self.assertEquals(False, gcs_helper.download_from_bucket(FakeGCSPath, 'some/file')) # Test with successful GET request. flexmock(gcs_helper).should_receive('gcs_get_request').\ and_return(FakeResponse2(200)) # Test with insufficient disk space. flexmock(json).should_receive('loads').and_return({"size": "2"}) flexmock(subprocess).should_receive('check_output').and_return( FakeDFOutput) flexmock(re).should_receive('sub').and_return(FakeFilteredDFOutput) self.assertEquals(False, gcs_helper.download_from_bucket(FakeGCSPath, 'some/file')) # Test with sufficient disk space. flexmock(json).should_receive('loads').and_return({ "size": "0", "mediaLink": "some/link"}) flexmock(subprocess).should_receive('check_output').and_return( FakeDFOutput) flexmock(re).should_receive('sub').and_return(FakeFilteredDFOutput) self.assertEquals(True, gcs_helper.download_from_bucket(FakeGCSPath, 'some/file'))
def restore_data(storage, path=''): """ Restores the Zookeeper snapshot. Args: storage: A str, one of the StorageTypes class members. path: A str, the name of the backup file to restore from. """ if storage not in StorageTypes().get_storage_types(): logging.error("Storage '{0}' not supported.") return False logging.info("Starting new zk restore.") if storage == StorageTypes.GCS: # Download backup file and store locally with a fixed name. if not gcs_helper.download_from_bucket(path, ZOOKEEPER_BACKUP_FILE_LOCATION): logging.error("Download from GCS failed. Aborting recovery...") return False # TODO Make sure there's a snapshot to rollback to if restore fails. # Not pressing for fresh deployments. flush_zk() try: backup_recovery_helper.untar_backup_files( ZOOKEEPER_BACKUP_FILE_LOCATION) except backup_exceptions.BRException as br_exception: logging.exception( "Error while unpacking backup files. Exception: {0}".format( str(br_exception))) if storage == StorageTypes.GCS: backup_recovery_helper.\ delete_local_backup_file(ZOOKEEPER_BACKUP_FILE_LOCATION) return False restore_zk(TMP_ZOOKEEPER_BACKUP) # Local cleanup. backup_recovery_helper.remove(TMP_ZOOKEEPER_BACKUP) if storage == StorageTypes.GCS: backup_recovery_helper.\ delete_local_backup_file(ZOOKEEPER_BACKUP_FILE_LOCATION) logging.info("Done with zk restore.") return True
def app_restore(storage, bucket_name=None): """ Restores the app source code from the backups location on the filesystem. Args: storage: A str, one of the StorageTypes class members. bucket_name: A str, the name of the bucket to restore apps from. Returns: True on success, False otherwise. """ # Create app backups dir if it doesn't exist. if not makedirs(APP_BACKUP_DIR_LOCATION): logging.warning( "Dir '{0}' already exists. Skipping dir creation...".format( APP_BACKUP_DIR_LOCATION)) # Download from GCS to backups location. if storage == StorageTypes.GCS: objects = gcs_helper.list_bucket(bucket_name) for app_path in objects: if not app_path.startswith(gcs_helper.APPS_GCS_PREFIX): continue # Only keep the relative name of the app file. # E.g. myapp.tar.gz (app_file) out of apps/myapp.tar.gz (app_path) app_file = app_path[len(gcs_helper.APPS_GCS_PREFIX):] source = 'gs://{0}/{1}'.format(bucket_name, app_path) destination = '{0}/{1}'.format(APP_BACKUP_DIR_LOCATION, app_file) if not gcs_helper.download_from_bucket(source, destination): logging.error( "Error while downloading '{0}' from GCS.".format(source)) delete_app_tars(APP_BACKUP_DIR_LOCATION) return False # Deploy apps. apps_to_deploy = [ os.path.join(APP_BACKUP_DIR_LOCATION, app) for app in os.listdir(APP_BACKUP_DIR_LOCATION) ] if not deploy_apps(apps_to_deploy): logging.error("Failed to successfully deploy one or more of the " "following apps: {0}".format(apps_to_deploy)) return False return True
def restore_data(storage, path=''): """ Restores the Zookeeper snapshot. Args: storage: A str, one of the StorageTypes class members. path: A str, the name of the backup file to restore from. """ if storage not in StorageTypes().get_storage_types(): logging.error("Storage '{0}' not supported.") return False logging.info("Starting new zk restore.") if storage == StorageTypes.GCS: # Download backup file and store locally with a fixed name. if not gcs_helper.download_from_bucket(path, ZOOKEEPER_BACKUP_FILE_LOCATION): logging.error("Download from GCS failed. Aborting recovery...") return False # TODO Make sure there's a snapshot to rollback to if restore fails. # Not pressing for fresh deployments. flush_zk() try: backup_recovery_helper.untar_backup_files(ZOOKEEPER_BACKUP_FILE_LOCATION) except backup_exceptions.BRException as br_exception: logging.exception("Error while unpacking backup files. Exception: {0}". format(str(br_exception))) if storage == StorageTypes.GCS: backup_recovery_helper.\ delete_local_backup_file(ZOOKEEPER_BACKUP_FILE_LOCATION) return False restore_zk(TMP_ZOOKEEPER_BACKUP) # Local cleanup. backup_recovery_helper.remove(TMP_ZOOKEEPER_BACKUP) if storage == StorageTypes.GCS: backup_recovery_helper.\ delete_local_backup_file(ZOOKEEPER_BACKUP_FILE_LOCATION) logging.info("Done with zk restore.") return True
def app_restore(storage, bucket_name=None): """ Restores the app source code from the backups location on the filesystem. Args: storage: A str, one of the StorageTypes class members. bucket_name: A str, the name of the bucket to restore apps from. Returns: True on success, False otherwise. """ # Create app backups dir if it doesn't exist. if not makedirs(APP_BACKUP_DIR_LOCATION): logging.warning("Dir '{0}' already exists. Skipping dir creation...". format(APP_BACKUP_DIR_LOCATION)) # Download from GCS to backups location. if storage == StorageTypes.GCS: objects = gcs_helper.list_bucket(bucket_name) for app_path in objects: if not app_path.startswith(gcs_helper.APPS_GCS_PREFIX): continue # Only keep the relative name of the app file. # E.g. myapp.tar.gz (app_file) out of apps/myapp.tar.gz (app_path) app_file = app_path[len(gcs_helper.APPS_GCS_PREFIX):] source = 'gs://{0}/{1}'.format(bucket_name, app_path) destination = '{0}/{1}'.format(APP_BACKUP_DIR_LOCATION, app_file) if not gcs_helper.download_from_bucket(source, destination): logging.error("Error while downloading '{0}' from GCS.".format(source)) delete_app_tars(APP_BACKUP_DIR_LOCATION) return False # Deploy apps. apps_to_deploy = [os.path.join(APP_BACKUP_DIR_LOCATION, app) for app in os.listdir(APP_BACKUP_DIR_LOCATION)] if not deploy_apps(apps_to_deploy): logging.error("Failed to successfully deploy one or more of the " "following apps: {0}".format(apps_to_deploy)) return False return True
def restore_data(storage, path=''): """ Restores the Zookeeper snapshot. Args: storage: A str, one of the StorageTypes class members. path: A str, the name of the backup file to restore from. """ if storage == StorageTypes.GCS: # Download backup file and store locally with a fixed name. if not gcs_helper.download_from_bucket(path, BACKUP_FILE_LOCATION): logging.error("Download from GCS failed. Aborting recovery...") return False # TODO Make sure there's a snapshot to rollback to if restore fails. # Not pressing for fresh deployments. if not shut_down_zookeeper.run(): logging.error("Unable to shut down Zookeeper. Aborting restore...") return False remove_old_data() try: untar_backup_files() except backup_exceptions.BRException as br_exception: logging.exception("Exception while restoring zk snapshots. Need to " "rollback... Exception: {0}".format( str(br_exception))) restore_snapshots() # Start Zookeeper. start_zookeeper.run() # Local cleanup. if storage == StorageTypes.GCS: remove_local_backup_file() logging.info("Done with restore.") return True
def restore_data(storage, path=''): """ Restores the Zookeeper snapshot. Args: storage: A str, one of the StorageTypes class members. path: A str, the name of the backup file to restore from. """ if storage == StorageTypes.GCS: # Download backup file and store locally with a fixed name. if not gcs_helper.download_from_bucket(path, BACKUP_FILE_LOCATION): logging.error("Download from GCS failed. Aborting recovery...") return False # TODO Make sure there's a snapshot to rollback to if restore fails. # Not pressing for fresh deployments. if not shut_down_zookeeper.run(): logging.error("Unable to shut down Zookeeper. Aborting restore...") return False remove_old_data() try: untar_backup_files() except backup_exceptions.BRException as br_exception: logging.exception("Exception while restoring zk snapshots. Need to " "rollback... Exception: {0}".format(str(br_exception))) restore_snapshots() # Start Zookeeper. start_zookeeper.run() # Local cleanup. if storage == StorageTypes.GCS: remove_local_backup_file() logging.info("Done with restore.") return True