def find_mysql_backup(replica_set, date, backup_type): """ Check whether or not a given replica set has a backup in S3 Args: replica_set: The replica set we're checking for. date: The date to search for. Returns: location: The location of the backup for this replica set. Returns None if not found. """ zk = host_utils.MysqlZookeeper() for repl_type in host_utils.REPLICA_TYPES: instance = zk.get_mysql_instance_from_replica_set( replica_set, repl_type) if instance: try: backup_file = backup.get_s3_backup(instance, date, backup_type) if backup_file: return backup_file break except boto.exception.S3ResponseError: raise except Exception as e: if backup.NO_BACKUP not in e[0]: raise return None
def find_mysql_backup(replica_set, date): """ Check whether or not a given replica set has a backup in S3 Args: replica_set: The replica set we're checking for. date: The date to search for. Returns: location: The location of the backup for this replica set. Returns None if not found. """ zk = host_utils.MysqlZookeeper() for repl_type in host_utils.REPLICA_TYPES: instance = zk.get_mysql_instance_from_replica_set(replica_set, repl_type) if instance: try: backup_file = backup.get_s3_backup(instance, date) if backup_file: return backup_file break except: # we'll get a 404 if there was no s3 backup, but that's OK, # so we can just move on to the next one. pass return None
def find_mysql_backup(replica_set, date, backup_type): """ Check whether or not a given replica set has a backup in S3 Args: replica_set: The replica set we're checking for. date: The date to search for. Returns: location: The location of the backup for this replica set. Returns None if not found. """ zk = host_utils.MysqlZookeeper() for repl_type in host_utils.REPLICA_TYPES: instance = zk.get_mysql_instance_from_replica_set(replica_set, repl_type) if instance: try: backup_file = backup.get_s3_backup(instance, date, backup_type) if backup_file: return backup_file break except boto.exception.S3ResponseError: raise except Exception as e: if backup.NO_BACKUP not in e[0]: raise return None
def find_mysql_backup(replica_set, date, backup_type): """ Check whether or not a given replica set has a backup in S3 Args: replica_set: The replica set we're checking for. date: The date to search for. Returns: location: The location of the backup for this replica set. Returns None if not found. """ zk = host_utils.MysqlZookeeper() for repl_type in host_utils.REPLICA_TYPES: instance = zk.get_mysql_instance_from_replica_set( replica_set, repl_type) if instance: try: backup_file = backup.get_s3_backup(instance, date, backup_type) if backup_file: return backup_file break except: # we'll get a 404 if there was no s3 backup, but that's OK, # so we can just move on to the next one. pass return None
def find_a_backup_to_restore(possible_sources, destination, backup_type, date=None): """ Based on supplied constains, try to find a backup to restore Args: source - A hostaddr object for where to pull a backup from destination - A hostaddr object for where to restore the backup backup_type - What sort of backup to restore date - What date should the backup be from Returns: restore_source - Where the backup was taken retore_file - Where the file exists on whichever storage restore_size - What is the size of the backup in bytes """ log.info('Possible source hosts:{}'.format(possible_sources)) if date: dates = [date] else: dates = [] for days in range(0, backup.DEFAULT_MAX_RESTORE_AGE): dates.append(datetime.date.today() - datetime.timedelta(days=days)) # Find a backup file with a preference for newer possible_keys = [] for restore_date in dates: if possible_keys: # we are looping to older dates, if we already found some keys, we # quit looking continue log.info('Looking for a backup for {}'.format(restore_date)) for possible_source in possible_sources: try: possible_keys.extend(backup.get_s3_backup(possible_source, str(restore_date), backup_type)) except boto.exception.S3ResponseError: raise except Exception as e: if backup.NO_BACKUP not in e[0]: raise log.info('No backup found on in s3 for host {source} ' 'on date {date}' ''.format(source=possible_source, date=restore_date)) if not possible_keys: raise Exception('Could not find a backup to restore') most_recent = None for key in possible_keys: if not most_recent: most_recent = key elif most_recent.last_modified < key.last_modified: most_recent = key log.info('Found a backup: {}'.format(key)) return most_recent
def find_a_backup_to_restore(restore_type, source, destination, date): """ Based on supplied constains, try to find a backup to restore Args: restore_type - What possible methods to use to pull the backup, options are 's3', 'remote_server'. source - A hostaddr object for where to pull a backup from destination - A hostaddr object for where to restore the backup date - What date should the backup be from Returns: restore_type - Which method to download a backup shoudl be used restore_source - Where the backup was taken retore_file - Where the file exists on whichever storage restore_size - What is the size of the backup in bytes """ zk = host_utils.MysqlZookeeper() possible_sources = set() if source: # the souce may not be in zk because it is a new replica set possible_sources.add(source) if source.get_zk_replica_set(): replica_set = source.get_zk_replica_set()[0] else: replica_set = None else: replica_set = destination.get_zk_replica_set()[0] for role in host_utils.REPLICA_TYPES: possible_sources.add(zk.get_mysql_instance_from_replica_set(replica_set, role)) log.info('Replica set detected as {replica_set}'.format(replica_set=replica_set)) log.info('Possible source hosts:{possible_sources}'.format(possible_sources=possible_sources)) if date: dates = [date] else: dates = [] for days in range(0, DEFAULT_MAX_RESTORE_AGE): dates.append(datetime.date.today() - datetime.timedelta(days=days)) # Find a backup file, preferably newer and less strong preferece on the master server restore_file = None for restore_date in dates: if restore_file: break log.info('Looking for a backup for {restore_date}'.format(restore_date=restore_date)) if not restore_type or restore_type == 's3': log.info('Looking for a backup in s3') for possible_source in possible_sources: try: (restore_file, restore_size) = backup.get_s3_backup(possible_source, str(restore_date)) restore_source = possible_source restore_type = 's3' break except: log.info('No backup found on in s3 for {source}'.format(source=possible_source)) if not restore_file: log.info('Could not find a backup in S3') # Perhaps the local filesystem of a remote servers? if not restore_type or restore_type == 'remote_server': log.info('Looking for a backup on remote filesystems') for possible_source in possible_sources: try: (restore_file, restore_size) = backup.get_remote_backup(possible_source, str(restore_date)) restore_source = possible_source restore_type = 'remote_server' break except: log.info('No backup found on remote filesystem for {source}'.format(source=possible_source)) if not restore_file: log.info('Could not find a backup on remote filesystems') if not restore_file: raise Exception('Could not find a backup to restore') log.info('Found a backup {restore_file} via ' '{restore_type}'.format(restore_file=restore_file, restore_type=restore_type)) (temp_path, target_path) = backup.get_paths(destination.port) return restore_type, restore_source, restore_file, restore_size
def find_a_backup_to_restore(possible_sources, destination, backup_type, date=None): """ Based on supplied constains, try to find a backup to restore Args: source - A hostaddr object for where to pull a backup from destination - A hostaddr object for where to restore the backup backup_type - What sort of backup to restore date - What date should the backup be from Returns: restore_source - Where the backup was taken retore_file - Where the file exists on whichever storage restore_size - What is the size of the backup in bytes """ log.info('Possible source hosts:{possible_sources}'.format(possible_sources=possible_sources)) if date: dates = [date] else: dates = [] for days in range(0, backup.DEFAULT_MAX_RESTORE_AGE): dates.append(datetime.date.today() - datetime.timedelta(days=days)) # Find a backup file with a preference for newer possible_keys = [] for restore_date in dates: if possible_keys: # we are looping to older dates, if we already found some keys, we # quit looking continue log.info('Looking for a backup for {restore_date}'.format(restore_date=restore_date)) for possible_source in possible_sources: try: possible_keys.extend(backup.get_s3_backup(possible_source, str(restore_date), backup_type)) except boto.exception.S3ResponseError: raise except Exception as e: if backup.NO_BACKUP not in e[0]: raise log.info('No backup found on in s3 for host {source} ' 'on date {date}' ''.format(source=possible_source, date=restore_date)) if not possible_keys: raise Exception('Could not find a backup to restore') most_recent = None for key in possible_keys: if not most_recent: most_recent = key elif most_recent.last_modified < key.last_modified: most_recent = key log.info('Found a backup: {}'.format(key)) return most_recent
def find_a_backup_to_restore(source, destination, date): """ Based on supplied constains, try to find a backup to restore Args: source - A hostaddr object for where to pull a backup from destination - A hostaddr object for where to restore the backup date - What date should the backup be from Returns: restore_source - Where the backup was taken retore_file - Where the file exists on whichever storage restore_size - What is the size of the backup in bytes """ zk = host_utils.MysqlZookeeper() possible_sources = list() if source: # the souce may not be in zk because it is a new replica set possible_sources.append(source) if source.get_zk_replica_set(): replica_set = source.get_zk_replica_set()[0] else: replica_set = None else: replica_set = destination.get_zk_replica_set()[0] for role in host_utils.REPLICA_TYPES: possible_sources.append(zk.get_mysql_instance_from_replica_set(replica_set, role)) log.info('Replica set detected as {replica_set}'.format(replica_set=replica_set)) log.info('Possible source hosts:{possible_sources}'.format(possible_sources=possible_sources)) if date: dates = [date] else: dates = [] for days in range(0, DEFAULT_MAX_RESTORE_AGE): dates.append(datetime.date.today() - datetime.timedelta(days=days)) # Find a backup file, preferably newer and less strong preferece on the master server restore_file = None for restore_date in dates: if restore_file: break log.info('Looking for a backup for {restore_date}'.format(restore_date=restore_date)) for possible_source in possible_sources: try: (restore_file, restore_size) = backup.get_s3_backup(possible_source, str(restore_date)) restore_source = possible_source break except: log.info('No backup found on in s3 for host {source} ' ' on date {date}' ''.format(source=possible_source, date=restore_date)) if not restore_file: raise Exception('Could not find a backup to restore') log.info('Found a backup: {restore_file}' ''.format(restore_file=restore_file)) return restore_source, restore_file, restore_size
def find_a_backup_to_restore(source, destination, date): """ Based on supplied constains, try to find a backup to restore Args: source - A hostaddr object for where to pull a backup from destination - A hostaddr object for where to restore the backup date - What date should the backup be from Returns: restore_source - Where the backup was taken retore_file - Where the file exists on whichever storage restore_size - What is the size of the backup in bytes """ zk = host_utils.MysqlZookeeper() possible_sources = list() if source: # the souce may not be in zk because it is a new replica set possible_sources.append(source) if source.get_zk_replica_set(): replica_set = source.get_zk_replica_set()[0] else: replica_set = None else: replica_set = destination.get_zk_replica_set()[0] for role in host_utils.REPLICA_TYPES: possible_sources.append( zk.get_mysql_instance_from_replica_set(replica_set, role)) log.info('Replica set detected as {replica_set}'.format( replica_set=replica_set)) log.info('Possible source hosts:{possible_sources}'.format( possible_sources=possible_sources)) if date: dates = [date] else: dates = [] for days in range(0, DEFAULT_MAX_RESTORE_AGE): dates.append(datetime.date.today() - datetime.timedelta(days=days)) # Find a backup file, preferably newer and less strong preferece on the master server restore_file = None for restore_date in dates: if restore_file: break log.info('Looking for a backup for {restore_date}'.format( restore_date=restore_date)) for possible_source in possible_sources: try: (restore_file, restore_size) = backup.get_s3_backup(possible_source, str(restore_date)) restore_source = possible_source break except: log.info('No backup found on in s3 for host {source} ' ' on date {date}' ''.format(source=possible_source, date=restore_date)) if not restore_file: raise Exception('Could not find a backup to restore') log.info('Found a backup: {restore_file}' ''.format(restore_file=restore_file)) return restore_source, restore_file, restore_size
def find_a_backup_to_restore(restore_type, source, destination, date): """ Based on supplied constains, try to find a backup to restore Args: restore_type - What possible methods to use to pull the backup, options are 's3', 'remote_server'. source - A hostaddr object for where to pull a backup from destination - A hostaddr object for where to restore the backup date - What date should the backup be from Returns: restore_type - Which method to download a backup shoudl be used restore_source - Where the backup was taken retore_file - Where the file exists on whichever storage restore_size - What is the size of the backup in bytes """ zk = host_utils.MysqlZookeeper() possible_sources = set() if source: # the souce may not be in zk because it is a new replica set possible_sources.add(source) if source.get_zk_replica_set(): replica_set = source.get_zk_replica_set()[0] else: replica_set = None else: replica_set = destination.get_zk_replica_set()[0] for role in host_utils.REPLICA_TYPES: possible_sources.add( zk.get_mysql_instance_from_replica_set(replica_set, role)) log.info('Replica set detected as {replica_set}'.format( replica_set=replica_set)) log.info('Possible source hosts:{possible_sources}'.format( possible_sources=possible_sources)) if date: dates = [date] else: dates = [] for days in range(0, DEFAULT_MAX_RESTORE_AGE): dates.append(datetime.date.today() - datetime.timedelta(days=days)) # Find a backup file, preferably newer and less strong preferece on the master server restore_file = None for restore_date in dates: if restore_file: break log.info('Looking for a backup for {restore_date}'.format( restore_date=restore_date)) if not restore_type or restore_type == 's3': log.info('Looking for a backup in s3') for possible_source in possible_sources: try: (restore_file, restore_size) = backup.get_s3_backup( possible_source, str(restore_date)) restore_source = possible_source restore_type = 's3' break except: log.info('No backup found on in s3 for {source}'.format( source=possible_source)) if not restore_file: log.info('Could not find a backup in S3') # Perhaps the local filesystem of a remote servers? if not restore_type or restore_type == 'remote_server': log.info('Looking for a backup on remote filesystems') for possible_source in possible_sources: try: (restore_file, restore_size) = backup.get_remote_backup( possible_source, str(restore_date)) restore_source = possible_source restore_type = 'remote_server' break except: log.info( 'No backup found on remote filesystem for {source}'. format(source=possible_source)) if not restore_file: log.info('Could not find a backup on remote filesystems') if not restore_file: raise Exception('Could not find a backup to restore') log.info('Found a backup {restore_file} via ' '{restore_type}'.format(restore_file=restore_file, restore_type=restore_type)) (temp_path, target_path) = backup.get_paths(destination.port) return restore_type, restore_source, restore_file, restore_size