Ejemplo n.º 1
0
    def __init__(self, host, logger):
        self._host = host
        self._log_helper = logger

        config_helper = Config_helper(host=self._host)

        self._backup_manager_host = config_helper.get_backup_manager_host()
        self._full_backup_day = config_helper.get_full_backup_day()

        self._remote_backup_cmd = config_helper.get_remote_backup_cmd()
        self._remote_cmd = Async_remote_command(
            host=self._host, command=self._remote_backup_cmd)

        self._backup_directory = os.path.join(config_helper.get_backup_dir(),
                                              self._host)
        self._backup_full_directory = os.path.join(self._backup_directory,
                                                   "full")
        self._backup_incremental_directory = os.path.join(
            self._backup_directory, "incremental")
Ejemplo n.º 2
0
    def __init__(self, host, logger):
        self._host = host
	self._log_helper = logger

        config_helper = Config_helper(host=self._host)

        self._backup_manager_host = config_helper.get_backup_manager_host()
        self._full_backup_day = config_helper.get_full_backup_day()

        self._remote_backup_cmd = config_helper.get_remote_backup_cmd()
        self._remote_cmd = Async_remote_command(host=self._host,
                                                command=self._remote_backup_cmd)

        self._backup_directory = os.path.join(config_helper.get_backup_dir(), self._host)
        self._backup_full_directory = os.path.join(self._backup_directory, "full")
        self._backup_incremental_directory = os.path.join(self._backup_directory, "incremental")
Ejemplo n.º 3
0
class Backup(object):
    CHECKPOINTS_FILE = "xtrabackup_checkpoints"
    BACKUP_TYPE_FULL = "full"
    BACKUP_TYPE_INC = "inc"

    def __init__(self, host, logger):
        self._host = host
        self._log_helper = logger

        config_helper = Config_helper(host=self._host)

        self._backup_manager_host = config_helper.get_backup_manager_host()
        self._full_backup_day = config_helper.get_full_backup_day()

        self._remote_backup_cmd = config_helper.get_remote_backup_cmd()
        self._remote_cmd = Async_remote_command(
            host=self._host, command=self._remote_backup_cmd)

        self._backup_directory = os.path.join(config_helper.get_backup_dir(),
                                              self._host)
        self._backup_full_directory = os.path.join(self._backup_directory,
                                                   "full")
        self._backup_incremental_directory = os.path.join(
            self._backup_directory, "incremental")

    def setup(self):
        self._log_helper.info_message(
            "Initializing backup directories and remote command...")

        if not os.path.isdir(self._backup_full_directory):
            os.makedirs(self._backup_full_directory)

        if not os.path.isdir(self._backup_incremental_directory):
            os.makedirs(self._backup_incremental_directory)

        self._remote_cmd.setup_remote_command()

    def is_full_backup_day(self):
        day_of_week = [
            'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday',
            'Sunday'
        ]
        current_time = time.localtime()
        today = day_of_week[current_time.tm_wday]

        if today == self._full_backup_day:
            return True

        return False

    def can_do_incremental_backup(self):
        # If today is a full backup day, and no prior full backups have been
        # done today, then full backup must be done
        if self.is_full_backup_day():
            full_backup_dt_list = self.get_directories_list(
                path=self._backup_full_directory)
            today = datetime.now()

            full_backup_done_today = False
            for dir_dt in full_backup_dt_list:
                if dir_dt.date() == today.date():
                    full_backup_done_today = True

            if not full_backup_done_today:
                return False

# If no backups are found, then full backup must be done
        latest_backup_dir_name = self.get_latest_backup_dir_name()
        if latest_backup_dir_name is None:
            return False

# If last backup was taken more than 1 day ago, then full backup must
# be done
        latest_backup_dir_name = latest_backup_dir_name.rstrip('/')
        latest_backup_date = datetime.strptime(
            os.path.basename(latest_backup_dir_name), "%Y_%m_%d_%H_%M_%S")
        timedelta = datetime.today() - latest_backup_date
        if timedelta.days > 1:
            return False

        return True

    def do_backup(self):
        if self.can_do_incremental_backup():
            return_val = self.do_incremental_backup()
        else:
            return_val = self.do_full_backup()

        return return_val

    def do_full_backup(self):
        self._log_helper.info_message("Starting FULL backup")

        backup_dir_name = os.path.join(self._backup_full_directory,
                                       time.strftime("%Y_%m_%d_%H_%M_%S"))
        if os.path.isdir(backup_dir_name):
            self._log_helper.error_message(
                "Backup directory %s already exists. Please move it away." %
                backup_dir_name)
            return False

        self._log_helper.info_message("Created backup directory %s" %
                                      backup_dir_name)

        os.makedirs(backup_dir_name)

        self._log_helper.info_message("Backup STARTED ...")

        backup_cmd_args = "-H %s -f -d %s" % (self._backup_manager_host,
                                              backup_dir_name)

        self._log_helper.info_message(
            "Executing remote command %s %s" %
            (self._remote_backup_cmd, backup_cmd_args))

        if self.execute_remote_backup_cmd(
                backup_cmd_args=backup_cmd_args) == False:
            return False

        self._log_helper.info_message(
            "Standardizing checkpoints file so it can be used for next run")
        if self.standardize_checkpoints_file(backup_dir_name) == False:
            self._log_helper.error_message(
                "Failed to standardize the checkpoints file, probably it does not exist"
            )
            return False

        self._log_helper.info_message("Backup COMPLETED")
        return {
            'backup_type': Backup.BACKUP_TYPE_FULL,
            'backup_dir': backup_dir_name
        }

    def do_incremental_backup(self):
        self._log_helper.info_message("Starting INCREMENTAL backup")

        latest_backup_dir_name = self.get_latest_backup_dir_name()

        self._log_helper.info_message("Last latest backup found at %s" %
                                      latest_backup_dir_name)

        last_lsn = self.get_last_lsn(latest_backup_dir_name)
        if last_lsn == False:
            self._log_helper.error_message(
                "Couldn't find LSN info from the latest backup dir %s" %
                latest_backup_dir_name)
            return False

        backup_dir_name = os.path.join(self._backup_incremental_directory,
                                       time.strftime("%Y_%m_%d_%H_%M_%S"))
        if os.path.isdir(backup_dir_name):
            self._log_helper.error_message(
                "Backup directory %s already exists. Please move it away." %
                backup_dir_name)
            return False

        self._log_helper.info_message("Created backup directory %s" %
                                      backup_dir_name)

        os.makedirs(backup_dir_name)

        self._log_helper.info_message("Backup STARTED from LSN %d ..." %
                                      last_lsn)

        backup_cmd_args = "-H %s -i -l %d -d %s" % (self._backup_manager_host,
                                                    last_lsn, backup_dir_name)

        self._log_helper.info_message(
            "Executing remote command %s %s" %
            (self._remote_backup_cmd, backup_cmd_args))

        if self.execute_remote_backup_cmd(
                backup_cmd_args=backup_cmd_args) == False:
            return False

        self._log_helper.info_message(
            "Standardizing checkpoints file so it can be used for next run")
        if self.standardize_checkpoints_file(backup_dir_name) == False:
            self._log_helper.error_message(
                "Failed to standardize the checkpoints file, probably it does not exist"
            )
            return False

        self._log_helper.info_message("Backup COMPLETED")
        return {
            'backup_type': Backup.BACKUP_TYPE_INC,
            'backup_dir': backup_dir_name
        }

    def execute_remote_backup_cmd(self, backup_cmd_args):
        if self._remote_cmd.execute_command(
                command_args=backup_cmd_args) == False:
            self._log_helper.error_message("Error executing remote backup "
                                           "script on %s" % self._host)
            return False

        cmd_result = self._remote_cmd.poll_command_result()

        if cmd_result['error'] == False:
            self._log_helper.info_message(
                "Backup FINISHED successfully [Duration: %s]" %
                cmd_result['duration'])
            return True

        self._log_helper.error_message("Backup FAILED")
        self._log_helper.error_message(cmd_result['error_msg'])
        return False

    def get_last_lsn(self, backup_dir):
        config = ConfigParser.RawConfigParser()
        try:
            config.read(os.path.join(backup_dir, Backup.CHECKPOINTS_FILE))
            return config.getint('xtrabackup_lsn_info', 'to_lsn')
        except Exception as e:
            return False

    def get_latest_backup_dir_name(self):
        full_backup_dir_list = self.get_directories_list(
            self._backup_full_directory)
        inc_backup_dir_list = self.get_directories_list(
            self._backup_incremental_directory)

        if len(full_backup_dir_list) == 0 and len(inc_backup_dir_list) == 0:
            return None

        if len(full_backup_dir_list) > 0:
            max_full_backup_day = max(full_backup_dir_list)
        if len(inc_backup_dir_list) > 0:
            max_inc_backup_day = max(inc_backup_dir_list)

        if (len(inc_backup_dir_list) == 0
                or max_full_backup_day > max_inc_backup_day):
            latest_backup_dir = os.path.join(
                self._backup_full_directory,
                max_full_backup_day.strftime("%Y_%m_%d_%H_%M_%S"))
        else:
            latest_backup_dir = os.path.join(
                self._backup_incremental_directory,
                max_inc_backup_day.strftime("%Y_%m_%d_%H_%M_%S"))
        return latest_backup_dir

    def get_directories_list(self, path):
        dir_list = []
        for root, dirs, files in os.walk(path):
            for dir in dirs:
                try:
                    dt = datetime.strptime(dir, "%Y_%m_%d_%H_%M_%S")
                    dir_list.append(dt)
                except Exception as e:
                    continue
        return dir_list

    def standardize_checkpoints_file(self, backup_dir):
        checkpoint_file = os.path.join(backup_dir, Backup.CHECKPOINTS_FILE)
        if os.path.isfile(checkpoint_file) == False:
            return False

        with open(checkpoint_file) as f:
            data = f.read()
        with open(checkpoint_file, 'w') as f:
            f.write("[xtrabackup_lsn_info]\n" + data)

        return True
Ejemplo n.º 4
0
class Backup(object):
    CHECKPOINTS_FILE = "xtrabackup_checkpoints"
    BACKUP_TYPE_FULL = "full"
    BACKUP_TYPE_INC = "inc"

    def __init__(self, host, logger):
        self._host = host
	self._log_helper = logger

        config_helper = Config_helper(host=self._host)

        self._backup_manager_host = config_helper.get_backup_manager_host()
        self._full_backup_day = config_helper.get_full_backup_day()

        self._remote_backup_cmd = config_helper.get_remote_backup_cmd()
        self._remote_cmd = Async_remote_command(host=self._host,
                                                command=self._remote_backup_cmd)

        self._backup_directory = os.path.join(config_helper.get_backup_dir(), self._host)
        self._backup_full_directory = os.path.join(self._backup_directory, "full")
        self._backup_incremental_directory = os.path.join(self._backup_directory, "incremental")

    def setup(self):
        self._log_helper.info_message("Initializing backup directories and remote command...")

        if not os.path.isdir(self._backup_full_directory):
            os.makedirs(self._backup_full_directory)

        if not os.path.isdir(self._backup_incremental_directory):
            os.makedirs(self._backup_incremental_directory)

        self._remote_cmd.setup_remote_command()
            
    def is_full_backup_day(self):
        day_of_week = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
        current_time = time.localtime()
        today = day_of_week[current_time.tm_wday]

        if today == self._full_backup_day:
            return True

        return False

    def can_do_incremental_backup(self):
	# If today is a full backup day, and no prior full backups have been
	# done today, then full backup must be done
        if self.is_full_backup_day():
	    full_backup_dt_list = self.get_directories_list(path=self._backup_full_directory)
	    today = datetime.now()

	    full_backup_done_today = False
	    for dir_dt in full_backup_dt_list:
		if dir_dt.date() == today.date():
		    full_backup_done_today = True

	    if not full_backup_done_today:
		return False

	# If no backups are found, then full backup must be done
        latest_backup_dir_name = self.get_latest_backup_dir_name()
        if latest_backup_dir_name is None:
            return False

	# If last backup was taken more than 1 day ago, then full backup must
	# be done
        latest_backup_dir_name = latest_backup_dir_name.rstrip('/')
        latest_backup_date = datetime.strptime(os.path.basename(latest_backup_dir_name), "%Y_%m_%d_%H_%M_%S")
        timedelta = datetime.today() - latest_backup_date
        if timedelta.days > 1:
            return False

        return True

    def do_backup(self):
        if self.can_do_incremental_backup():
            return_val = self.do_incremental_backup()
        else:
            return_val = self.do_full_backup()

        return return_val

    def do_full_backup(self):
        self._log_helper.info_message("Starting FULL backup")

        backup_dir_name = os.path.join(self._backup_full_directory, time.strftime("%Y_%m_%d_%H_%M_%S"))
        if os.path.isdir(backup_dir_name):
            self._log_helper.error_message("Backup directory %s already exists. Please move it away." % backup_dir_name)
            return False

        self._log_helper.info_message("Created backup directory %s" % backup_dir_name)

        os.makedirs(backup_dir_name)

        self._log_helper.info_message("Backup STARTED ...")

        backup_cmd_args = "-H %s -f -d %s" % (self._backup_manager_host, backup_dir_name)

        self._log_helper.info_message("Executing remote command %s %s" % (self._remote_backup_cmd, backup_cmd_args))

        if self.execute_remote_backup_cmd(backup_cmd_args=backup_cmd_args) == False:
            return False

        self._log_helper.info_message("Standardizing checkpoints file so it can be used for next run")
        if self.standardize_checkpoints_file(backup_dir_name) == False:
            self._log_helper.error_message("Failed to standardize the checkpoints file, probably it does not exist")
            return False

        self._log_helper.info_message("Backup COMPLETED")
        return {'backup_type': Backup.BACKUP_TYPE_FULL, 'backup_dir': backup_dir_name}

    def do_incremental_backup(self):
        self._log_helper.info_message("Starting INCREMENTAL backup")

        latest_backup_dir_name = self.get_latest_backup_dir_name()

        self._log_helper.info_message("Last latest backup found at %s" % latest_backup_dir_name)

        last_lsn = self.get_last_lsn(latest_backup_dir_name)
        if last_lsn == False:
            self._log_helper.error_message("Couldn't find LSN info from the latest backup dir %s" % latest_backup_dir_name)
            return False

        backup_dir_name = os.path.join(self._backup_incremental_directory, time.strftime("%Y_%m_%d_%H_%M_%S"))
        if os.path.isdir(backup_dir_name):
            self._log_helper.error_message("Backup directory %s already exists. Please move it away." % backup_dir_name)
            return False

        self._log_helper.info_message("Created backup directory %s" % backup_dir_name)

        os.makedirs(backup_dir_name)

        self._log_helper.info_message("Backup STARTED from LSN %d ..." % last_lsn)

        backup_cmd_args = "-H %s -i -l %d -d %s" % (self._backup_manager_host, last_lsn, backup_dir_name)

        self._log_helper.info_message("Executing remote command %s %s" % (self._remote_backup_cmd, backup_cmd_args))

        if self.execute_remote_backup_cmd(backup_cmd_args=backup_cmd_args) == False:
            return False

        self._log_helper.info_message("Standardizing checkpoints file so it can be used for next run")
        if self.standardize_checkpoints_file(backup_dir_name) == False:
            self._log_helper.error_message("Failed to standardize the checkpoints file, probably it does not exist")
            return False

        self._log_helper.info_message("Backup COMPLETED")
        return {'backup_type': Backup.BACKUP_TYPE_INC, 'backup_dir': backup_dir_name}

    def execute_remote_backup_cmd(self, backup_cmd_args):
        if self._remote_cmd.execute_command(command_args=backup_cmd_args) == False:
            self._log_helper.error_message("Error executing remote backup "
					    "script on %s" % self._host)
            return False

        cmd_result = self._remote_cmd.poll_command_result()

        if cmd_result['error'] == False:
            self._log_helper.info_message("Backup FINISHED successfully [Duration: %s]" % cmd_result['duration'])
            return True

        self._log_helper.error_message("Backup FAILED")
        self._log_helper.error_message(cmd_result['error_msg'])
        return False

    def get_last_lsn(self, backup_dir):
        config = ConfigParser.RawConfigParser()
        try:
            config.read(os.path.join(backup_dir, Backup.CHECKPOINTS_FILE))
            return config.getint('xtrabackup_lsn_info', 'to_lsn')
        except Exception as e:
            return False

    def get_latest_backup_dir_name(self):
        full_backup_dir_list = self.get_directories_list(self._backup_full_directory)
        inc_backup_dir_list = self.get_directories_list(self._backup_incremental_directory)

        if len(full_backup_dir_list) == 0 and len(inc_backup_dir_list) == 0:
            return None

        if len(full_backup_dir_list) > 0:
            max_full_backup_day = max(full_backup_dir_list)
        if len(inc_backup_dir_list) > 0:
            max_inc_backup_day = max(inc_backup_dir_list)

        if (len(inc_backup_dir_list) == 0 or max_full_backup_day > max_inc_backup_day):
            latest_backup_dir = os.path.join(self._backup_full_directory,
                                            max_full_backup_day.strftime("%Y_%m_%d_%H_%M_%S"))
        else:
            latest_backup_dir = os.path.join(self._backup_incremental_directory,
                                            max_inc_backup_day.strftime("%Y_%m_%d_%H_%M_%S"))
        return latest_backup_dir

    def get_directories_list(self, path):
        dir_list = []
        for root, dirs, files in os.walk(path):
            for dir in dirs:
                try:
                    dt = datetime.strptime(dir, "%Y_%m_%d_%H_%M_%S")
                    dir_list.append(dt)
                except Exception as e:
                    continue
        return dir_list

    def standardize_checkpoints_file(self, backup_dir):
        checkpoint_file = os.path.join(backup_dir, Backup.CHECKPOINTS_FILE)
        if os.path.isfile(checkpoint_file) == False:
            return False

        with open(checkpoint_file) as f:
            data = f.read()
        with open(checkpoint_file, 'w') as f:
            f.write("[xtrabackup_lsn_info]\n" + data)

        return True