def find_mysql_backup(replica_set, date, backup_type):
""" Check whether or not a given replica set has a backup in S3
Args:
replica_set: The replica set we're checking for.
date: The date to search for.
Returns:
location: The location of the backup for this replica set.
Returns None if not found.
"""
zk = host_utils.MysqlZookeeper()
for repl_type in host_utils.REPLICA_TYPES:
instance = zk.get_mysql_instance_from_replica_set(
replica_set, repl_type)
if instance:
try:
backup_file = backup.get_s3_backup(instance, date, backup_type)
if backup_file:
return backup_file
break
except boto.exception.S3ResponseError:
raise
except Exception as e:
if backup.NO_BACKUP not in e[0]:
raise
return None
def find_mysql_backup(replica_set, date):
""" Check whether or not a given replica set has a backup in S3
Args:
replica_set: The replica set we're checking for.
date: The date to search for.
Returns:
location: The location of the backup for this replica set.
Returns None if not found.
"""
zk = host_utils.MysqlZookeeper()
for repl_type in host_utils.REPLICA_TYPES:
instance = zk.get_mysql_instance_from_replica_set(replica_set,
repl_type)
if instance:
try:
backup_file = backup.get_s3_backup(instance, date)
if backup_file:
return backup_file
break
except:
# we'll get a 404 if there was no s3 backup, but that's OK,
# so we can just move on to the next one.
pass
return None
def find_mysql_backup(replica_set, date, backup_type):
""" Check whether or not a given replica set has a backup in S3
Args:
replica_set: The replica set we're checking for.
date: The date to search for.
Returns:
location: The location of the backup for this replica set.
Returns None if not found.
"""
zk = host_utils.MysqlZookeeper()
for repl_type in host_utils.REPLICA_TYPES:
instance = zk.get_mysql_instance_from_replica_set(replica_set,
repl_type)
if instance:
try:
backup_file = backup.get_s3_backup(instance, date, backup_type)
if backup_file:
return backup_file
break
except boto.exception.S3ResponseError:
raise
except Exception as e:
if backup.NO_BACKUP not in e[0]:
raise
return None
def find_mysql_backup(replica_set, date, backup_type):
""" Check whether or not a given replica set has a backup in S3
Args:
replica_set: The replica set we're checking for.
date: The date to search for.
Returns:
location: The location of the backup for this replica set.
Returns None if not found.
"""
zk = host_utils.MysqlZookeeper()
for repl_type in host_utils.REPLICA_TYPES:
instance = zk.get_mysql_instance_from_replica_set(
replica_set, repl_type)
if instance:
try:
backup_file = backup.get_s3_backup(instance, date, backup_type)
if backup_file:
return backup_file
break
except:
# we'll get a 404 if there was no s3 backup, but that's OK,
# so we can just move on to the next one.
pass
return None
def find_a_backup_to_restore(possible_sources, destination,
backup_type, date=None):
""" Based on supplied constains, try to find a backup to restore
Args:
source - A hostaddr object for where to pull a backup from
destination - A hostaddr object for where to restore the backup
backup_type - What sort of backup to restore
date - What date should the backup be from
Returns:
restore_source - Where the backup was taken
retore_file - Where the file exists on whichever storage
restore_size - What is the size of the backup in bytes
"""
log.info('Possible source hosts:{}'.format(possible_sources))
if date:
dates = [date]
else:
dates = []
for days in range(0, backup.DEFAULT_MAX_RESTORE_AGE):
dates.append(datetime.date.today() - datetime.timedelta(days=days))
# Find a backup file with a preference for newer
possible_keys = []
for restore_date in dates:
if possible_keys:
# we are looping to older dates, if we already found some keys, we
# quit looking
continue
log.info('Looking for a backup for {}'.format(restore_date))
for possible_source in possible_sources:
try:
possible_keys.extend(backup.get_s3_backup(possible_source,
str(restore_date),
backup_type))
except boto.exception.S3ResponseError:
raise
except Exception as e:
if backup.NO_BACKUP not in e[0]:
raise
log.info('No backup found on in s3 for host {source} '
'on date {date}'
''.format(source=possible_source,
date=restore_date))
if not possible_keys:
raise Exception('Could not find a backup to restore')
most_recent = None
for key in possible_keys:
if not most_recent:
most_recent = key
elif most_recent.last_modified < key.last_modified:
most_recent = key
log.info('Found a backup: {}'.format(key))
return most_recent
def find_a_backup_to_restore(restore_type, source, destination, date):
""" Based on supplied constains, try to find a backup to restore
Args:
restore_type - What possible methods to use to pull the backup,
options are 's3', 'remote_server'.
source - A hostaddr object for where to pull a backup from
destination - A hostaddr object for where to restore the backup
date - What date should the backup be from
Returns:
restore_type - Which method to download a backup shoudl be used
restore_source - Where the backup was taken
retore_file - Where the file exists on whichever storage
restore_size - What is the size of the backup in bytes
"""
zk = host_utils.MysqlZookeeper()
possible_sources = set()
if source:
# the souce may not be in zk because it is a new replica set
possible_sources.add(source)
if source.get_zk_replica_set():
replica_set = source.get_zk_replica_set()[0]
else:
replica_set = None
else:
replica_set = destination.get_zk_replica_set()[0]
for role in host_utils.REPLICA_TYPES:
possible_sources.add(zk.get_mysql_instance_from_replica_set(replica_set, role))
log.info('Replica set detected as {replica_set}'.format(replica_set=replica_set))
log.info('Possible source hosts:{possible_sources}'.format(possible_sources=possible_sources))
if date:
dates = [date]
else:
dates = []
for days in range(0, DEFAULT_MAX_RESTORE_AGE):
dates.append(datetime.date.today() - datetime.timedelta(days=days))
# Find a backup file, preferably newer and less strong preferece on the master server
restore_file = None
for restore_date in dates:
if restore_file:
break
log.info('Looking for a backup for {restore_date}'.format(restore_date=restore_date))
if not restore_type or restore_type == 's3':
log.info('Looking for a backup in s3')
for possible_source in possible_sources:
try:
(restore_file, restore_size) = backup.get_s3_backup(possible_source, str(restore_date))
restore_source = possible_source
restore_type = 's3'
break
except:
log.info('No backup found on in s3 for {source}'.format(source=possible_source))
if not restore_file:
log.info('Could not find a backup in S3')
# Perhaps the local filesystem of a remote servers?
if not restore_type or restore_type == 'remote_server':
log.info('Looking for a backup on remote filesystems')
for possible_source in possible_sources:
try:
(restore_file, restore_size) = backup.get_remote_backup(possible_source, str(restore_date))
restore_source = possible_source
restore_type = 'remote_server'
break
except:
log.info('No backup found on remote filesystem for {source}'.format(source=possible_source))
if not restore_file:
log.info('Could not find a backup on remote filesystems')
if not restore_file:
raise Exception('Could not find a backup to restore')
log.info('Found a backup {restore_file} via '
'{restore_type}'.format(restore_file=restore_file,
restore_type=restore_type))
(temp_path, target_path) = backup.get_paths(destination.port)
return restore_type, restore_source, restore_file, restore_size
def find_a_backup_to_restore(possible_sources, destination,
backup_type, date=None):
""" Based on supplied constains, try to find a backup to restore
Args:
source - A hostaddr object for where to pull a backup from
destination - A hostaddr object for where to restore the backup
backup_type - What sort of backup to restore
date - What date should the backup be from
Returns:
restore_source - Where the backup was taken
retore_file - Where the file exists on whichever storage
restore_size - What is the size of the backup in bytes
"""
log.info('Possible source hosts:{possible_sources}'.format(possible_sources=possible_sources))
if date:
dates = [date]
else:
dates = []
for days in range(0, backup.DEFAULT_MAX_RESTORE_AGE):
dates.append(datetime.date.today() - datetime.timedelta(days=days))
# Find a backup file with a preference for newer
possible_keys = []
for restore_date in dates:
if possible_keys:
# we are looping to older dates, if we already found some keys, we
# quit looking
continue
log.info('Looking for a backup for {restore_date}'.format(restore_date=restore_date))
for possible_source in possible_sources:
try:
possible_keys.extend(backup.get_s3_backup(possible_source,
str(restore_date),
backup_type))
except boto.exception.S3ResponseError:
raise
except Exception as e:
if backup.NO_BACKUP not in e[0]:
raise
log.info('No backup found on in s3 for host {source} '
'on date {date}'
''.format(source=possible_source,
date=restore_date))
if not possible_keys:
raise Exception('Could not find a backup to restore')
most_recent = None
for key in possible_keys:
if not most_recent:
most_recent = key
elif most_recent.last_modified < key.last_modified:
most_recent = key
log.info('Found a backup: {}'.format(key))
return most_recent
def find_a_backup_to_restore(source, destination, date):
""" Based on supplied constains, try to find a backup to restore
Args:
source - A hostaddr object for where to pull a backup from
destination - A hostaddr object for where to restore the backup
date - What date should the backup be from
Returns:
restore_source - Where the backup was taken
retore_file - Where the file exists on whichever storage
restore_size - What is the size of the backup in bytes
"""
zk = host_utils.MysqlZookeeper()
possible_sources = list()
if source:
# the souce may not be in zk because it is a new replica set
possible_sources.append(source)
if source.get_zk_replica_set():
replica_set = source.get_zk_replica_set()[0]
else:
replica_set = None
else:
replica_set = destination.get_zk_replica_set()[0]
for role in host_utils.REPLICA_TYPES:
possible_sources.append(zk.get_mysql_instance_from_replica_set(replica_set, role))
log.info('Replica set detected as {replica_set}'.format(replica_set=replica_set))
log.info('Possible source hosts:{possible_sources}'.format(possible_sources=possible_sources))
if date:
dates = [date]
else:
dates = []
for days in range(0, DEFAULT_MAX_RESTORE_AGE):
dates.append(datetime.date.today() - datetime.timedelta(days=days))
# Find a backup file, preferably newer and less strong preferece on the master server
restore_file = None
for restore_date in dates:
if restore_file:
break
log.info('Looking for a backup for {restore_date}'.format(restore_date=restore_date))
for possible_source in possible_sources:
try:
(restore_file, restore_size) = backup.get_s3_backup(possible_source, str(restore_date))
restore_source = possible_source
break
except:
log.info('No backup found on in s3 for host {source} '
' on date {date}'
''.format(source=possible_source,
date=restore_date))
if not restore_file:
raise Exception('Could not find a backup to restore')
log.info('Found a backup: {restore_file}'
''.format(restore_file=restore_file))
return restore_source, restore_file, restore_size
def find_a_backup_to_restore(source, destination, date):
""" Based on supplied constains, try to find a backup to restore
Args:
source - A hostaddr object for where to pull a backup from
destination - A hostaddr object for where to restore the backup
date - What date should the backup be from
Returns:
restore_source - Where the backup was taken
retore_file - Where the file exists on whichever storage
restore_size - What is the size of the backup in bytes
"""
zk = host_utils.MysqlZookeeper()
possible_sources = list()
if source:
# the souce may not be in zk because it is a new replica set
possible_sources.append(source)
if source.get_zk_replica_set():
replica_set = source.get_zk_replica_set()[0]
else:
replica_set = None
else:
replica_set = destination.get_zk_replica_set()[0]
for role in host_utils.REPLICA_TYPES:
possible_sources.append(
zk.get_mysql_instance_from_replica_set(replica_set, role))
log.info('Replica set detected as {replica_set}'.format(
replica_set=replica_set))
log.info('Possible source hosts:{possible_sources}'.format(
possible_sources=possible_sources))
if date:
dates = [date]
else:
dates = []
for days in range(0, DEFAULT_MAX_RESTORE_AGE):
dates.append(datetime.date.today() - datetime.timedelta(days=days))
# Find a backup file, preferably newer and less strong preferece on the master server
restore_file = None
for restore_date in dates:
if restore_file:
break
log.info('Looking for a backup for {restore_date}'.format(
restore_date=restore_date))
for possible_source in possible_sources:
try:
(restore_file,
restore_size) = backup.get_s3_backup(possible_source,
str(restore_date))
restore_source = possible_source
break
except:
log.info('No backup found on in s3 for host {source} '
' on date {date}'
''.format(source=possible_source, date=restore_date))
if not restore_file:
raise Exception('Could not find a backup to restore')
log.info('Found a backup: {restore_file}'
''.format(restore_file=restore_file))
return restore_source, restore_file, restore_size
def find_a_backup_to_restore(restore_type, source, destination, date):
""" Based on supplied constains, try to find a backup to restore
Args:
restore_type - What possible methods to use to pull the backup,
options are 's3', 'remote_server'.
source - A hostaddr object for where to pull a backup from
destination - A hostaddr object for where to restore the backup
date - What date should the backup be from
Returns:
restore_type - Which method to download a backup shoudl be used
restore_source - Where the backup was taken
retore_file - Where the file exists on whichever storage
restore_size - What is the size of the backup in bytes
"""
zk = host_utils.MysqlZookeeper()
possible_sources = set()
if source:
# the souce may not be in zk because it is a new replica set
possible_sources.add(source)
if source.get_zk_replica_set():
replica_set = source.get_zk_replica_set()[0]
else:
replica_set = None
else:
replica_set = destination.get_zk_replica_set()[0]
for role in host_utils.REPLICA_TYPES:
possible_sources.add(
zk.get_mysql_instance_from_replica_set(replica_set, role))
log.info('Replica set detected as {replica_set}'.format(
replica_set=replica_set))
log.info('Possible source hosts:{possible_sources}'.format(
possible_sources=possible_sources))
if date:
dates = [date]
else:
dates = []
for days in range(0, DEFAULT_MAX_RESTORE_AGE):
dates.append(datetime.date.today() - datetime.timedelta(days=days))
# Find a backup file, preferably newer and less strong preferece on the master server
restore_file = None
for restore_date in dates:
if restore_file:
break
log.info('Looking for a backup for {restore_date}'.format(
restore_date=restore_date))
if not restore_type or restore_type == 's3':
log.info('Looking for a backup in s3')
for possible_source in possible_sources:
try:
(restore_file, restore_size) = backup.get_s3_backup(
possible_source, str(restore_date))
restore_source = possible_source
restore_type = 's3'
break
except:
log.info('No backup found on in s3 for {source}'.format(
source=possible_source))
if not restore_file:
log.info('Could not find a backup in S3')
# Perhaps the local filesystem of a remote servers?
if not restore_type or restore_type == 'remote_server':
log.info('Looking for a backup on remote filesystems')
for possible_source in possible_sources:
try:
(restore_file, restore_size) = backup.get_remote_backup(
possible_source, str(restore_date))
restore_source = possible_source
restore_type = 'remote_server'
break
except:
log.info(
'No backup found on remote filesystem for {source}'.
format(source=possible_source))
if not restore_file:
log.info('Could not find a backup on remote filesystems')
if not restore_file:
raise Exception('Could not find a backup to restore')
log.info('Found a backup {restore_file} via '
'{restore_type}'.format(restore_file=restore_file,
restore_type=restore_type))
(temp_path, target_path) = backup.get_paths(destination.port)
return restore_type, restore_source, restore_file, restore_size
