def __init__(self, host, logger):
self._host = host
self._log_helper = logger
config_helper = Config_helper(host=self._host)
self._backup_manager_host = config_helper.get_backup_manager_host()
self._full_backup_day = config_helper.get_full_backup_day()
self._remote_backup_cmd = config_helper.get_remote_backup_cmd()
self._remote_cmd = Async_remote_command(
host=self._host, command=self._remote_backup_cmd)
self._backup_directory = os.path.join(config_helper.get_backup_dir(),
self._host)
self._backup_full_directory = os.path.join(self._backup_directory,
"full")
self._backup_incremental_directory = os.path.join(
self._backup_directory, "incremental")
def __init__(self, host, logger):
self._host = host
self._log_helper = logger
config_helper = Config_helper(host=self._host)
self._backup_manager_host = config_helper.get_backup_manager_host()
self._full_backup_day = config_helper.get_full_backup_day()
self._remote_backup_cmd = config_helper.get_remote_backup_cmd()
self._remote_cmd = Async_remote_command(host=self._host,
command=self._remote_backup_cmd)
self._backup_directory = os.path.join(config_helper.get_backup_dir(), self._host)
self._backup_full_directory = os.path.join(self._backup_directory, "full")
self._backup_incremental_directory = os.path.join(self._backup_directory, "incremental")
class Backup(object):
CHECKPOINTS_FILE = "xtrabackup_checkpoints"
BACKUP_TYPE_FULL = "full"
BACKUP_TYPE_INC = "inc"
def __init__(self, host, logger):
self._host = host
self._log_helper = logger
config_helper = Config_helper(host=self._host)
self._backup_manager_host = config_helper.get_backup_manager_host()
self._full_backup_day = config_helper.get_full_backup_day()
self._remote_backup_cmd = config_helper.get_remote_backup_cmd()
self._remote_cmd = Async_remote_command(
host=self._host, command=self._remote_backup_cmd)
self._backup_directory = os.path.join(config_helper.get_backup_dir(),
self._host)
self._backup_full_directory = os.path.join(self._backup_directory,
"full")
self._backup_incremental_directory = os.path.join(
self._backup_directory, "incremental")
def setup(self):
self._log_helper.info_message(
"Initializing backup directories and remote command...")
if not os.path.isdir(self._backup_full_directory):
os.makedirs(self._backup_full_directory)
if not os.path.isdir(self._backup_incremental_directory):
os.makedirs(self._backup_incremental_directory)
self._remote_cmd.setup_remote_command()
def is_full_backup_day(self):
day_of_week = [
'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday',
'Sunday'
]
current_time = time.localtime()
today = day_of_week[current_time.tm_wday]
if today == self._full_backup_day:
return True
return False
def can_do_incremental_backup(self):
# If today is a full backup day, and no prior full backups have been
# done today, then full backup must be done
if self.is_full_backup_day():
full_backup_dt_list = self.get_directories_list(
path=self._backup_full_directory)
today = datetime.now()
full_backup_done_today = False
for dir_dt in full_backup_dt_list:
if dir_dt.date() == today.date():
full_backup_done_today = True
if not full_backup_done_today:
return False
# If no backups are found, then full backup must be done
latest_backup_dir_name = self.get_latest_backup_dir_name()
if latest_backup_dir_name is None:
return False
# If last backup was taken more than 1 day ago, then full backup must
# be done
latest_backup_dir_name = latest_backup_dir_name.rstrip('/')
latest_backup_date = datetime.strptime(
os.path.basename(latest_backup_dir_name), "%Y_%m_%d_%H_%M_%S")
timedelta = datetime.today() - latest_backup_date
if timedelta.days > 1:
return False
return True
def do_backup(self):
if self.can_do_incremental_backup():
return_val = self.do_incremental_backup()
else:
return_val = self.do_full_backup()
return return_val
def do_full_backup(self):
self._log_helper.info_message("Starting FULL backup")
backup_dir_name = os.path.join(self._backup_full_directory,
time.strftime("%Y_%m_%d_%H_%M_%S"))
if os.path.isdir(backup_dir_name):
self._log_helper.error_message(
"Backup directory %s already exists. Please move it away." %
backup_dir_name)
return False
self._log_helper.info_message("Created backup directory %s" %
backup_dir_name)
os.makedirs(backup_dir_name)
self._log_helper.info_message("Backup STARTED ...")
backup_cmd_args = "-H %s -f -d %s" % (self._backup_manager_host,
backup_dir_name)
self._log_helper.info_message(
"Executing remote command %s %s" %
(self._remote_backup_cmd, backup_cmd_args))
if self.execute_remote_backup_cmd(
backup_cmd_args=backup_cmd_args) == False:
return False
self._log_helper.info_message(
"Standardizing checkpoints file so it can be used for next run")
if self.standardize_checkpoints_file(backup_dir_name) == False:
self._log_helper.error_message(
"Failed to standardize the checkpoints file, probably it does not exist"
)
return False
self._log_helper.info_message("Backup COMPLETED")
return {
'backup_type': Backup.BACKUP_TYPE_FULL,
'backup_dir': backup_dir_name
}
def do_incremental_backup(self):
self._log_helper.info_message("Starting INCREMENTAL backup")
latest_backup_dir_name = self.get_latest_backup_dir_name()
self._log_helper.info_message("Last latest backup found at %s" %
latest_backup_dir_name)
last_lsn = self.get_last_lsn(latest_backup_dir_name)
if last_lsn == False:
self._log_helper.error_message(
"Couldn't find LSN info from the latest backup dir %s" %
latest_backup_dir_name)
return False
backup_dir_name = os.path.join(self._backup_incremental_directory,
time.strftime("%Y_%m_%d_%H_%M_%S"))
if os.path.isdir(backup_dir_name):
self._log_helper.error_message(
"Backup directory %s already exists. Please move it away." %
backup_dir_name)
return False
self._log_helper.info_message("Created backup directory %s" %
backup_dir_name)
os.makedirs(backup_dir_name)
self._log_helper.info_message("Backup STARTED from LSN %d ..." %
last_lsn)
backup_cmd_args = "-H %s -i -l %d -d %s" % (self._backup_manager_host,
last_lsn, backup_dir_name)
self._log_helper.info_message(
"Executing remote command %s %s" %
(self._remote_backup_cmd, backup_cmd_args))
if self.execute_remote_backup_cmd(
backup_cmd_args=backup_cmd_args) == False:
return False
self._log_helper.info_message(
"Standardizing checkpoints file so it can be used for next run")
if self.standardize_checkpoints_file(backup_dir_name) == False:
self._log_helper.error_message(
"Failed to standardize the checkpoints file, probably it does not exist"
)
return False
self._log_helper.info_message("Backup COMPLETED")
return {
'backup_type': Backup.BACKUP_TYPE_INC,
'backup_dir': backup_dir_name
}
def execute_remote_backup_cmd(self, backup_cmd_args):
if self._remote_cmd.execute_command(
command_args=backup_cmd_args) == False:
self._log_helper.error_message("Error executing remote backup "
"script on %s" % self._host)
return False
cmd_result = self._remote_cmd.poll_command_result()
if cmd_result['error'] == False:
self._log_helper.info_message(
"Backup FINISHED successfully [Duration: %s]" %
cmd_result['duration'])
return True
self._log_helper.error_message("Backup FAILED")
self._log_helper.error_message(cmd_result['error_msg'])
return False
def get_last_lsn(self, backup_dir):
config = ConfigParser.RawConfigParser()
try:
config.read(os.path.join(backup_dir, Backup.CHECKPOINTS_FILE))
return config.getint('xtrabackup_lsn_info', 'to_lsn')
except Exception as e:
return False
def get_latest_backup_dir_name(self):
full_backup_dir_list = self.get_directories_list(
self._backup_full_directory)
inc_backup_dir_list = self.get_directories_list(
self._backup_incremental_directory)
if len(full_backup_dir_list) == 0 and len(inc_backup_dir_list) == 0:
return None
if len(full_backup_dir_list) > 0:
max_full_backup_day = max(full_backup_dir_list)
if len(inc_backup_dir_list) > 0:
max_inc_backup_day = max(inc_backup_dir_list)
if (len(inc_backup_dir_list) == 0
or max_full_backup_day > max_inc_backup_day):
latest_backup_dir = os.path.join(
self._backup_full_directory,
max_full_backup_day.strftime("%Y_%m_%d_%H_%M_%S"))
else:
latest_backup_dir = os.path.join(
self._backup_incremental_directory,
max_inc_backup_day.strftime("%Y_%m_%d_%H_%M_%S"))
return latest_backup_dir
def get_directories_list(self, path):
dir_list = []
for root, dirs, files in os.walk(path):
for dir in dirs:
try:
dt = datetime.strptime(dir, "%Y_%m_%d_%H_%M_%S")
dir_list.append(dt)
except Exception as e:
continue
return dir_list
def standardize_checkpoints_file(self, backup_dir):
checkpoint_file = os.path.join(backup_dir, Backup.CHECKPOINTS_FILE)
if os.path.isfile(checkpoint_file) == False:
return False
with open(checkpoint_file) as f:
data = f.read()
with open(checkpoint_file, 'w') as f:
f.write("[xtrabackup_lsn_info]\n" + data)
return True
class Backup(object):
CHECKPOINTS_FILE = "xtrabackup_checkpoints"
BACKUP_TYPE_FULL = "full"
BACKUP_TYPE_INC = "inc"
def __init__(self, host, logger):
self._host = host
self._log_helper = logger
config_helper = Config_helper(host=self._host)
self._backup_manager_host = config_helper.get_backup_manager_host()
self._full_backup_day = config_helper.get_full_backup_day()
self._remote_backup_cmd = config_helper.get_remote_backup_cmd()
self._remote_cmd = Async_remote_command(host=self._host,
command=self._remote_backup_cmd)
self._backup_directory = os.path.join(config_helper.get_backup_dir(), self._host)
self._backup_full_directory = os.path.join(self._backup_directory, "full")
self._backup_incremental_directory = os.path.join(self._backup_directory, "incremental")
def setup(self):
self._log_helper.info_message("Initializing backup directories and remote command...")
if not os.path.isdir(self._backup_full_directory):
os.makedirs(self._backup_full_directory)
if not os.path.isdir(self._backup_incremental_directory):
os.makedirs(self._backup_incremental_directory)
self._remote_cmd.setup_remote_command()
def is_full_backup_day(self):
day_of_week = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
current_time = time.localtime()
today = day_of_week[current_time.tm_wday]
if today == self._full_backup_day:
return True
return False
def can_do_incremental_backup(self):
# If today is a full backup day, and no prior full backups have been
# done today, then full backup must be done
if self.is_full_backup_day():
full_backup_dt_list = self.get_directories_list(path=self._backup_full_directory)
today = datetime.now()
full_backup_done_today = False
for dir_dt in full_backup_dt_list:
if dir_dt.date() == today.date():
full_backup_done_today = True
if not full_backup_done_today:
return False
# If no backups are found, then full backup must be done
latest_backup_dir_name = self.get_latest_backup_dir_name()
if latest_backup_dir_name is None:
return False
# If last backup was taken more than 1 day ago, then full backup must
# be done
latest_backup_dir_name = latest_backup_dir_name.rstrip('/')
latest_backup_date = datetime.strptime(os.path.basename(latest_backup_dir_name), "%Y_%m_%d_%H_%M_%S")
timedelta = datetime.today() - latest_backup_date
if timedelta.days > 1:
return False
return True
def do_backup(self):
if self.can_do_incremental_backup():
return_val = self.do_incremental_backup()
else:
return_val = self.do_full_backup()
return return_val
def do_full_backup(self):
self._log_helper.info_message("Starting FULL backup")
backup_dir_name = os.path.join(self._backup_full_directory, time.strftime("%Y_%m_%d_%H_%M_%S"))
if os.path.isdir(backup_dir_name):
self._log_helper.error_message("Backup directory %s already exists. Please move it away." % backup_dir_name)
return False
self._log_helper.info_message("Created backup directory %s" % backup_dir_name)
os.makedirs(backup_dir_name)
self._log_helper.info_message("Backup STARTED ...")
backup_cmd_args = "-H %s -f -d %s" % (self._backup_manager_host, backup_dir_name)
self._log_helper.info_message("Executing remote command %s %s" % (self._remote_backup_cmd, backup_cmd_args))
if self.execute_remote_backup_cmd(backup_cmd_args=backup_cmd_args) == False:
return False
self._log_helper.info_message("Standardizing checkpoints file so it can be used for next run")
if self.standardize_checkpoints_file(backup_dir_name) == False:
self._log_helper.error_message("Failed to standardize the checkpoints file, probably it does not exist")
return False
self._log_helper.info_message("Backup COMPLETED")
return {'backup_type': Backup.BACKUP_TYPE_FULL, 'backup_dir': backup_dir_name}
def do_incremental_backup(self):
self._log_helper.info_message("Starting INCREMENTAL backup")
latest_backup_dir_name = self.get_latest_backup_dir_name()
self._log_helper.info_message("Last latest backup found at %s" % latest_backup_dir_name)
last_lsn = self.get_last_lsn(latest_backup_dir_name)
if last_lsn == False:
self._log_helper.error_message("Couldn't find LSN info from the latest backup dir %s" % latest_backup_dir_name)
return False
backup_dir_name = os.path.join(self._backup_incremental_directory, time.strftime("%Y_%m_%d_%H_%M_%S"))
if os.path.isdir(backup_dir_name):
self._log_helper.error_message("Backup directory %s already exists. Please move it away." % backup_dir_name)
return False
self._log_helper.info_message("Created backup directory %s" % backup_dir_name)
os.makedirs(backup_dir_name)
self._log_helper.info_message("Backup STARTED from LSN %d ..." % last_lsn)
backup_cmd_args = "-H %s -i -l %d -d %s" % (self._backup_manager_host, last_lsn, backup_dir_name)
self._log_helper.info_message("Executing remote command %s %s" % (self._remote_backup_cmd, backup_cmd_args))
if self.execute_remote_backup_cmd(backup_cmd_args=backup_cmd_args) == False:
return False
self._log_helper.info_message("Standardizing checkpoints file so it can be used for next run")
if self.standardize_checkpoints_file(backup_dir_name) == False:
self._log_helper.error_message("Failed to standardize the checkpoints file, probably it does not exist")
return False
self._log_helper.info_message("Backup COMPLETED")
return {'backup_type': Backup.BACKUP_TYPE_INC, 'backup_dir': backup_dir_name}
def execute_remote_backup_cmd(self, backup_cmd_args):
if self._remote_cmd.execute_command(command_args=backup_cmd_args) == False:
self._log_helper.error_message("Error executing remote backup "
"script on %s" % self._host)
return False
cmd_result = self._remote_cmd.poll_command_result()
if cmd_result['error'] == False:
self._log_helper.info_message("Backup FINISHED successfully [Duration: %s]" % cmd_result['duration'])
return True
self._log_helper.error_message("Backup FAILED")
self._log_helper.error_message(cmd_result['error_msg'])
return False
def get_last_lsn(self, backup_dir):
config = ConfigParser.RawConfigParser()
try:
config.read(os.path.join(backup_dir, Backup.CHECKPOINTS_FILE))
return config.getint('xtrabackup_lsn_info', 'to_lsn')
except Exception as e:
return False
def get_latest_backup_dir_name(self):
full_backup_dir_list = self.get_directories_list(self._backup_full_directory)
inc_backup_dir_list = self.get_directories_list(self._backup_incremental_directory)
if len(full_backup_dir_list) == 0 and len(inc_backup_dir_list) == 0:
return None
if len(full_backup_dir_list) > 0:
max_full_backup_day = max(full_backup_dir_list)
if len(inc_backup_dir_list) > 0:
max_inc_backup_day = max(inc_backup_dir_list)
if (len(inc_backup_dir_list) == 0 or max_full_backup_day > max_inc_backup_day):
latest_backup_dir = os.path.join(self._backup_full_directory,
max_full_backup_day.strftime("%Y_%m_%d_%H_%M_%S"))
else:
latest_backup_dir = os.path.join(self._backup_incremental_directory,
max_inc_backup_day.strftime("%Y_%m_%d_%H_%M_%S"))
return latest_backup_dir
def get_directories_list(self, path):
dir_list = []
for root, dirs, files in os.walk(path):
for dir in dirs:
try:
dt = datetime.strptime(dir, "%Y_%m_%d_%H_%M_%S")
dir_list.append(dt)
except Exception as e:
continue
return dir_list
def standardize_checkpoints_file(self, backup_dir):
checkpoint_file = os.path.join(backup_dir, Backup.CHECKPOINTS_FILE)
if os.path.isfile(checkpoint_file) == False:
return False
with open(checkpoint_file) as f:
data = f.read()
with open(checkpoint_file, 'w') as f:
f.write("[xtrabackup_lsn_info]\n" + data)
return True