def inc_backup(self): # Taking Incremental backup recent_bck = self.recent_full_backup_file() recent_inc = self.recent_inc_backup_file() check_env_obj = CheckEnv() product_type = check_env_obj.check_mysql_product() # Creating time-stamped incremental backup directory inc_backup_dir = self.create_backup_directory(self.inc_dir) # Checking if there is any incremental backup if recent_inc == 0: # If there is no incremental backup # If you have a question why we check whether MariaDB or MySQL installed? # See BUG -> https://bugs.launchpad.net/percona-xtrabackup/+bug/1444541 if product_type == 2: # Taking incremental backup with MariaDB. (--incremental-force-scan option will be added for BUG workaround) args = "%s --defaults-file=%s --user=%s --password='******' " \ "--incremental-force-scan --incremental %s --incremental-basedir %s/%s" % \ (self.backup_tool, self.mycnf, self.mysql_user, self.mysql_password, self.inc_dir, self.full_dir, recent_bck) elif product_type == 3: # Taking incremental backup with MySQL. args = "%s --defaults-file=%s --user=%s --password='******' " \ "--target-dir=%s --incremental-basedir=%s/%s --backup" % \ (self.backup_tool, self.mycnf, self.mysql_user, self.mysql_password, inc_backup_dir, self.full_dir, recent_bck) if hasattr(self, 'mysql_socket'): args += " --socket=%s" % (self.mysql_socket) elif hasattr(self, 'mysql_host') and hasattr(self, 'mysql_port'): args += " --host=%s" % self.mysql_host args += " --port=%s" % self.mysql_port else: logger.critical( "Neither mysql_socket nor mysql_host and mysql_port are defined in config!" ) return False # Adding compression support for incremental backup if hasattr(self, 'compress'): args += " --compress=%s" % (self.compress) if hasattr(self, 'compress_chunk_size'): args += " --compress-chunk-size=%s" % ( self.compress_chunk_size) if hasattr(self, 'compress_threads'): args += " --compress-threads=%s" % (self.compress_threads) # Adding encryption support for incremental backup if hasattr(self, 'encrypt'): args += " --encrypt=%s" % (self.encrypt) if hasattr(self, 'encrypt_key'): args += " --encrypt-key=%s" % (self.encrypt_key) if hasattr(self, 'encrypt_key_file'): args += " --encrypt_key_file=%s" % (self.encrypt_key_file) if hasattr(self, 'encrypt_threads'): args += " --encrypt-threads=%s" % (self.encrypt_threads) if hasattr(self, 'encrypt_chunk_size'): args += " --encrypt-chunk-size=%s" % (self.encrypt_chunk_size) if 'encrypt' in args: logger.debug("Applying workaround for LP #1444255") xbcrypt_command = "%s -d -k %s -a %s -i %s/%s/xtrabackup_checkpoints.xbcrypt " \ "-o %s/%s/xtrabackup_checkpoints" % \ (self.xbcrypt, self.encrypt_key, self.encrypt, self.full_dir, recent_bck, self.full_dir, recent_bck) logger.debug( "The following xbcrypt command will be executed %s", xbcrypt_command) status, output = subprocess.getstatusoutput(xbcrypt_command) if status == 0: logger.debug(output[-27:]) else: logger.error("XBCRYPT COMMAND FAILED!") time.sleep(5) logger.error(output) return False logger.debug("The following backup command will be executed %s", args) status, output = subprocess.getstatusoutput(args) if status == 0: logger.debug(output[-27:]) return True else: logger.error("INCREMENT BACKUP FAILED!") time.sleep(5) logger.error(output) return False else: # If there is already existing incremental backup if product_type == 2: # Taking incremental backup with MariaDB. (--incremental-force-scan option will be added for BUG workaround) args = "%s --defaults-file=%s --user=%s --password='******' " \ "--incremental-force-scan --incremental %s --incremental-basedir %s/%s" % \ (self.backup_tool, self.mycnf, self.mysql_user, self.mysql_password, self.inc_dir, self.inc_dir, recent_inc) elif product_type == 3: args = "%s --defaults-file=%s --user=%s --password='******' " \ "--target-dir=%s --incremental-basedir=%s/%s --backup" % \ (self.backup_tool, self.mycnf, self.mysql_user, self.mysql_password, inc_backup_dir, self.inc_dir, recent_inc) if hasattr(self, 'mysql_socket'): args += " --socket=%s" % (self.mysql_socket) elif hasattr(self, 'mysql_host') and hasattr(self, 'mysql_port'): args += " --host=%s" % self.mysql_host args += " --port=%s" % self.mysql_port else: logger.critical( "Neither mysql_socket nor mysql_host and mysql_port are defined in config!" ) return False # Adding compression support for incremental backup if hasattr(self, 'compress'): args += " --compress=%s" % (self.compress) if hasattr(self, 'compress_chunk_size'): args += " --compress_chunk_size=%s" % ( self.compress_chunk_size) if hasattr(self, 'compress-threads'): args += " --compress_threads=%s" % (self.compress_threads) # Adding encryption support for incremental backup if hasattr(self, 'encrypt'): args += " --encrypt=%s" % (self.encrypt) if hasattr(self, 'encrypt_key'): args += " --encrypt-key=%s" % (self.encrypt_key) if hasattr(self, 'encrypt_key_file'): args += " --encrypt-key-file=%s" % (self.encrypt_key_file) if hasattr(self, 'encrypt_threads'): args += " --encrypt-threads=%s" % (self.encrypt_threads) if hasattr(self, 'encrypt_chunk_size'): args += " --encrypt-chunk-size=%s" % (self.encrypt_chunk_size) if 'encrypt' in args: logger.debug("Applying workaround for LP #1444255") xbcrypt_command = "%s -d -k %s -a %s -i %s/%s/xtrabackup_checkpoints.xbcrypt " \ "-o %s/%s/xtrabackup_checkpoints" % \ (self.xbcrypt, self.encrypt_key, self.encrypt, self.inc_dir, recent_inc, self.inc_dir, recent_inc) logger.debug( "The following xbcrypt command will be executed %s", xbcrypt_command) status, output = subprocess.getstatusoutput(xbcrypt_command) if status == 0: logger.debug(output[-27:]) else: logger.error("XBCRYPT COMMAND FAILED!") time.sleep(5) logger.error(output) return False logger.debug("The following backup command will be executed %s", args) status, output = subprocess.getstatusoutput(args) if status == 0: logger.debug(output[-27:]) return True else: logger.error("INCREMENT BACKUP FAILED!") time.sleep(5) logger.error(output) return False
def inc_backup(self): # Taking Incremental backup recent_bck = self.recent_full_backup_file() recent_inc = self.recent_inc_backup_file() check_env_obj = CheckEnv(self.conf) product_type = check_env_obj.check_mysql_product() # Creating time-stamped incremental backup directory inc_backup_dir = self.create_backup_directory(self.inc_dir) # Checking if there is any incremental backup if recent_inc == 0: # If there is no incremental backup # If you have a question why we check whether MariaDB or MySQL installed? # See BUG -> # https://bugs.launchpad.net/percona-xtrabackup/+bug/1444541 if product_type == 2: # Taking incremental backup with MariaDB. # (--incremental-force-scan option will be added for BUG # workaround) args = "%s --defaults-file=%s --user=%s --password='******' " \ "--incremental-force-scan --incremental %s --incremental-basedir %s/%s" % \ (self.backup_tool, self.mycnf, self.mysql_user, self.mysql_password, self.inc_dir, self.full_dir, recent_bck) elif product_type == 3: # Taking incremental backup with MySQL. args = "%s --defaults-file=%s --user=%s --password='******' " \ "--target-dir=%s --incremental-basedir=%s/%s --backup" % \ (self.backup_tool, self.mycnf, self.mysql_user, self.mysql_password, inc_backup_dir, self.full_dir, recent_bck) if hasattr(self, 'mysql_socket'): args += " --socket=%s" % (self.mysql_socket) elif hasattr(self, 'mysql_host') and hasattr(self, 'mysql_port'): args += " --host=%s" % self.mysql_host args += " --port=%s" % self.mysql_port else: logger.critical( "Neither mysql_socket nor mysql_host and mysql_port are not defined in config!" ) return False # Adding compression support for incremental backup if hasattr(self, 'compress'): args += " --compress=%s" % (self.compress) if hasattr(self, 'compress_chunk_size'): args += " --compress-chunk-size=%s" % ( self.compress_chunk_size) if hasattr(self, 'compress_threads'): args += " --compress-threads=%s" % (self.compress_threads) # Adding encryption support for incremental backup if hasattr(self, 'encrypt'): args += " --encrypt=%s" % (self.encrypt) if hasattr(self, 'encrypt_key'): args += " --encrypt-key=%s" % (self.encrypt_key) if hasattr(self, 'encrypt_key_file'): args += " --encrypt_key_file=%s" % (self.encrypt_key_file) if hasattr(self, 'encrypt_threads'): args += " --encrypt-threads=%s" % (self.encrypt_threads) if hasattr(self, 'encrypt_chunk_size'): args += " --encrypt-chunk-size=%s" % (self.encrypt_chunk_size) # Extract and decrypt streamed full backup prior to executing incremental backup if hasattr(self, 'stream') \ and hasattr(self, 'encrypt') \ and hasattr(self, 'xbs_decrypt'): logger.debug( "Using xbstream to extract and decrypt from full_backup.stream!" ) xbstream_command = "%s %s --decrypt=%s --encrypt-key=%s --encrypt-threads=%s " \ "< %s/%s/full_backup.stream -C %s/%s" % ( self.xbstream, self.xbstream_options, self.decrypt, self.encrypt_key, self.encrypt_threads, self.full_dir, recent_bck, self.full_dir, recent_bck ) logger.debug( "The following xbstream command will be executed %s", xbstream_command) if self.dry == 0 and isfile(("%s/%s/full_backup.stream") % (self.full_dir, recent_bck)): status, output = subprocess.getstatusoutput( xbstream_command) if status == 0: logger.debug("XBSTREAM command succeeded.") else: logger.error("XBSTREAM COMMAND FAILED!") time.sleep(5) logger.error(output) return False # Extract streamed full backup prior to executing incremental backup elif hasattr(self, 'stream'): logger.debug( "Using xbstream to extract from full_backup.stream!") xbstream_command = "%s %s < %s/%s/full_backup.stream -C %s/%s" % ( self.xbstream, self.xbstream_options, self.full_dir, recent_bck, self.full_dir, recent_bck) logger.debug( "The following xbstream command will be executed %s", xbstream_command) if self.dry == 0 and isfile(("%s/%s/full_backup.stream") % (self.full_dir, recent_bck)): status, output = subprocess.getstatusoutput( xbstream_command) if status == 0: logger.debug("XBSTREAM command succeeded.") else: logger.error("XBSTREAM COMMAND FAILED!") time.sleep(5) logger.error(output) return False elif 'encrypt' in args: logger.debug("Applying workaround for LP #1444255") xbcrypt_command = "%s -d -k %s -a %s -i %s/%s/xtrabackup_checkpoints.xbcrypt " \ "-o %s/%s/xtrabackup_checkpoints" % \ (self.xbcrypt, self.encrypt_key, self.encrypt, self.full_dir, recent_bck, self.full_dir, recent_bck) logger.debug( "The following xbcrypt command will be executed %s", xbcrypt_command) if self.dry == 0: status, output = subprocess.getstatusoutput( xbcrypt_command) if status == 0: logger.debug(output[-27:]) else: logger.error("XBCRYPT COMMAND FAILED!") time.sleep(5) logger.error(output) return False # Checking if extra options were passed: if hasattr(self, 'xtra_options'): args += " " args += self.xtra_options # Checking if partial recovery list is available if hasattr(self, 'partial_list'): args += " " args += '--databases="%s"' % (self.partial_list) logger.warning("Partial Backup is enabled!") # Checking if streaming enabled for backups if hasattr(self, 'stream'): args += " " args += '--stream="%s"' % self.stream args += " > %s/inc_backup.stream" % inc_backup_dir logger.warning("Streaming is enabled!") logger.debug("The following backup command will be executed %s", args) if self.dry == 0: status, output = subprocess.getstatusoutput(args) if status == 0: logger.debug(output) #logger.debug(output[-27:]) return True else: logger.error("INCREMENT BACKUP FAILED!") time.sleep(5) logger.error(output) return False else: # If there is already existing incremental backup if product_type == 2: # Taking incremental backup with MariaDB. # (--incremental-force-scan option will be added for BUG # workaround) args = "%s --defaults-file=%s --user=%s --password='******' " \ "--incremental-force-scan --incremental %s --incremental-basedir %s/%s" % \ (self.backup_tool, self.mycnf, self.mysql_user, self.mysql_password, self.inc_dir, self.inc_dir, recent_inc) elif product_type == 3: args = "%s --defaults-file=%s --user=%s --password='******' " \ "--target-dir=%s --incremental-basedir=%s/%s --backup" % \ (self.backup_tool, self.mycnf, self.mysql_user, self.mysql_password, inc_backup_dir, self.inc_dir, recent_inc) if hasattr(self, 'mysql_socket'): args += " --socket=%s" % (self.mysql_socket) elif hasattr(self, 'mysql_host') and hasattr(self, 'mysql_port'): args += " --host=%s" % self.mysql_host args += " --port=%s" % self.mysql_port else: logger.critical( "Neither mysql_socket nor mysql_host and mysql_port are defined in config!" ) return False # Adding compression support for incremental backup if hasattr(self, 'compress'): args += " --compress=%s" % (self.compress) if hasattr(self, 'compress_chunk_size'): args += " --compress_chunk_size=%s" % ( self.compress_chunk_size) if hasattr(self, 'compress-threads'): args += " --compress_threads=%s" % (self.compress_threads) # Adding encryption support for incremental backup if hasattr(self, 'encrypt'): args += " --encrypt=%s" % (self.encrypt) if hasattr(self, 'encrypt_key'): args += " --encrypt-key=%s" % (self.encrypt_key) if hasattr(self, 'encrypt_key_file'): args += " --encrypt-key-file=%s" % (self.encrypt_key_file) if hasattr(self, 'encrypt_threads'): args += " --encrypt-threads=%s" % (self.encrypt_threads) if hasattr(self, 'encrypt_chunk_size'): args += " --encrypt-chunk-size=%s" % (self.encrypt_chunk_size) # Extract and decrypt streamed full backup prior to executing incremental backup if hasattr(self, 'stream') \ and hasattr(self, 'encrypt') \ and hasattr(self, 'xbs_decrypt'): logger.debug( "Using xbstream to extract and decrypt from inc_backup.stream!" ) xbstream_command = "%s %s --decrypt=%s --encrypt-key=%s --encrypt-threads=%s " \ "< %s/%s/inc_backup.stream -C %s/%s" % ( self.xbstream, self.xbstream_options, self.decrypt, self.encrypt_key, self.encrypt_threads, self.inc_dir, recent_inc, self.inc_dir, recent_inc ) logger.debug( "The following xbstream command will be executed %s", xbstream_command) if self.dry == 0 and isfile( ("%s/%s/inc_backup.stream") % (self.inc_dir, recent_inc)): status, output = subprocess.getstatusoutput( xbstream_command) if status == 0: logger.debug("XBSTREAM command succeeded.") else: logger.error("XBSTREAM COMMAND FAILED!") time.sleep(5) logger.error(output) return False # Extracting streamed incremental backup prior to executing new incremental backup elif hasattr(self, 'stream'): logger.debug( "Using xbstream to extract from inc_backup.stream!") xbstream_command = "%s %s < %s/%s/inc_backup.stream -C %s/%s" % ( self.xbstream, self.xbstream_options, self.inc_dir, recent_inc, self.inc_dir, recent_inc) logger.debug( "The following xbstream command will be executed %s", xbstream_command) if self.dry == 0 and isfile( ("%s/%s/inc_backup.stream") % (self.full_dir, recent_bck)): status, output = subprocess.getstatusoutput( xbstream_command) if status == 0: logger.debug("XBSTREAM command succeeded.") else: logger.error("XBSTREAM COMMAND FAILED!") time.sleep(5) logger.error(output) return False elif 'encrypt' in args: logger.debug("Applying workaround for LP #1444255") xbcrypt_command = "%s -d -k %s -a %s -i %s/%s/xtrabackup_checkpoints.xbcrypt " \ "-o %s/%s/xtrabackup_checkpoints" % \ (self.xbcrypt, self.encrypt_key, self.encrypt, self.inc_dir, recent_inc, self.inc_dir, recent_inc) logger.debug( "The following xbcrypt command will be executed %s", xbcrypt_command) if self.dry == 0: status, output = subprocess.getstatusoutput( xbcrypt_command) if status == 0: logger.debug(output[-27:]) else: logger.error("XBCRYPT COMMAND FAILED!") time.sleep(5) logger.error(output) return False # Checking if extra options were passed: if hasattr(self, 'xtra_options'): args += " " args += self.xtra_options # Checking if partial recovery list is available if hasattr(self, 'partial_list'): args += " " args += '--databases="%s"' % (self.partial_list) logger.warning("Partial Backup is enabled!") # Checking if streaming enabled for backups if hasattr(self, 'stream'): args += " " args += '--stream="%s"' % self.stream args += " > %s/inc_backup.stream" % inc_backup_dir logger.warning("Streaming is enabled!") logger.debug("The following backup command will be executed %s", args) if self.dry == 0: status, output = subprocess.getstatusoutput(args) if status == 0: logger.debug(output) #logger.debug(output[-27:]) return True else: logger.error("INCREMENT BACKUP FAILED!") time.sleep(5) logger.error(output) return False