def add_tag(
self, backup_type: str, backup_size: Optional[str], backup_status: Optional[str]
) -> bool:
"""
Method for adding backup tags
:param backup_type: The backup type - Full/Inc
:param backup_size: The size of the backup in human readable format
:param backup_status: Status: OK or Status: Failed
:return: True if no exception
"""
# skip tagging unless self.tag
if not self.tag:
logger.info("TAGGING SKIPPED")
return True
# Currently only support Inc and Full types, calculate name based on this
assert backup_type in (
"Full",
"Inc",
), "add_tag(): backup_type {}: must be 'Full' or 'Inc'".format(backup_type)
backup_name = (
helpers.get_latest_dir_name(
str(self.builder_obj.backup_options.get("full_dir"))
)
if backup_type == "Full"
else helpers.get_latest_dir_name(
str(self.builder_obj.backup_options.get("inc_dir"))
)
)
# Calculate more tag fields, create string
backup_timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
backup_tag_str = (
"{bk_name}\t{bk_type}\t{bk_status}\t{bk_timestamp}\t{bk_size}\t'{bk_tag}'\n"
)
# Apply tag
with open(
"{}/backup_tags.txt".format(
self.builder_obj.backup_options.get("backup_dir")
),
"a",
) as backup_tags_file:
backup_tag_final = backup_tag_str.format(
bk_name=backup_name,
bk_type=backup_type,
bk_status=backup_status,
bk_timestamp=backup_timestamp,
bk_size=backup_size,
bk_tag=self.tag,
)
backup_tags_file.write(backup_tag_final)
return True
def prepare_only_full_backup(self) -> Union[None, bool, Exception]:
recent_bck = helpers.get_latest_dir_name(
str(self.prepare_options.backup_options.get("full_dir")))
backup_builder = BackupBuilderChecker(self.conf, dry_run=self.dry)
if recent_bck:
apply_log_only = None
if not os.listdir(
str(self.prepare_options.backup_options.get("inc_dir"))):
logger.info("- - - - Preparing Full Backup - - - -")
self.prepare_options.untar_backup(recent_bck=recent_bck)
# Extracting/decrypting from streamed backup and extra checks goes here
backup_builder.extract_decrypt_from_stream_backup(
recent_full_bck=recent_bck)
else:
logger.info(
"- - - - Preparing Full backup for incrementals - - - -")
logger.info(
"- - - - Final prepare,will occur after preparing all inc backups - - - -"
)
time.sleep(3)
apply_log_only = True
# Prepare command
backup_prepare_cmd = self.prepare_options.prepare_command_builder(
full_backup=recent_bck, apply_log_only=apply_log_only)
self.run_prepare_command(
str(self.prepare_options.backup_options.get("full_dir")),
recent_bck,
backup_prepare_cmd,
)
return True
def test_get_latest_dir_name(self):
os.makedirs("tests/DELETE_ME", mode=777, exist_ok=True)
os.makedirs("tests/DELETE_ME/2021-05-06_11-48-31", mode=777, exist_ok=True)
os.makedirs("tests/DELETE_ME/2021-05-06_11-47-31", mode=777, exist_ok=True)
assert (
helpers.get_latest_dir_name(path=f"{os.path.dirname(__file__)}/DELETE_ME")
== "2021-05-06_11-48-31"
)
def prepare_inc_full_backups(self) -> Union[None, bool, Exception]:
backup_builder = BackupBuilderChecker(self.conf, dry_run=self.dry)
if not os.listdir(
str(self.prepare_options.backup_options.get("inc_dir"))):
logger.info(
"- - - - You have no Incremental backups. So will prepare only latest Full backup - - - -"
)
return self.prepare_only_full_backup()
else:
logger.info("- - - - You have Incremental backups. - - - -")
recent_bck = helpers.get_latest_dir_name(
str(self.prepare_options.backup_options.get("full_dir")))
if self.prepare_only_full_backup():
logger.info("Preparing Incs: ")
list_of_dir = sorted(
os.listdir(
str(self.prepare_options.backup_options.get(
"inc_dir"))))
for inc_backup_dir in list_of_dir:
apply_log_only = None
if inc_backup_dir != max(
os.listdir(
str(
self.prepare_options.backup_options.get(
"inc_dir")))):
logger.info(
"Preparing Incremental backups in sequence. Incremental backup dir/name is {}"
.format(inc_backup_dir))
apply_log_only = True
else:
logger.info(
"Preparing last Incremental backup, inc backup dir/name is {}"
.format(inc_backup_dir))
# Extracting/decrypting from streamed backup and extra checks goes here
backup_builder.extract_decrypt_from_stream_backup(
recent_inc_bck=inc_backup_dir, flag=True)
# Prepare command
backup_prepare_cmd = self.prepare_options.prepare_command_builder(
full_backup=recent_bck,
incremental=inc_backup_dir,
apply_log_only=apply_log_only,
)
self.run_prepare_command(
str(self.prepare_options.backup_options.get(
"inc_dir")),
inc_backup_dir,
backup_prepare_cmd,
)
logger.info("- - - - The end of the Prepare Stage. - - - -")
return True
def run_xtra_copyback(self,
data_dir: Optional[str] = None) -> Optional[bool]:
# Running Xtrabackup with --copy-back option
copy_back = "{} --copy-back {} --target-dir={}/{} --data_dir={}".format(
self.backup_options.get("backup_tool"),
self.backup_options.get("xtra_options"),
self.backup_options.get("full_dir"),
helpers.get_latest_dir_name(
str(self.backup_options.get("full_dir"))),
self.mysql_options.get("data_dir")
if data_dir is None else data_dir,
)
return ProcessRunner.run_command(copy_back)
def last_full_backup_date(
self, path: Optional[str] = None, full_backup_interval: Optional[float] = None
) -> bool:
"""
Check if last full backup date retired or not.
:return: True if last full backup date older than given interval, False if it is newer.
"""
# Finding last full backup date from dir/folder name
full_dir = path or str(self.builder_obj.backup_options.get("full_dir"))
backup_interval = full_backup_interval or str(
self.builder_obj.backup_options.get("full_backup_interval")
)
max_dir = helpers.get_latest_dir_name(full_dir)
dir_date = datetime.strptime(str(max_dir), "%Y-%m-%d_%H-%M-%S")
now = datetime.now()
return float((now - dir_date).total_seconds()) >= float(backup_interval)
def copy_back_action(self,
options: Optional[str] = None) -> Optional[bool]:
"""
Function for complete recover/copy-back actions
:return: True if succeeded. Error if failed.
"""
try:
self.check_if_backup_prepared(
str(self.backup_options.get("full_dir")),
helpers.get_latest_dir_name(
str(self.backup_options.get("full_dir"))),
)
self.shutdown_mysql()
if self.move_data_dir() and self.copy(options=options):
logger.info("All data copied back successfully. ")
logger.info("Your MySQL server is UP again")
return True
except Exception as err:
logger.error("{}: {}".format(type(err).__name__, err))
return None
def prepare_with_tags(self) -> Optional[bool]:
# Method for preparing backups based on passed backup tags
found_backups = BackupPrepareBuilderChecker.parse_backup_tags(
backup_dir=str(
self.prepare_options.backup_options.get("backup_dir")),
tag_name=self.tag,
)
recent_bck = helpers.get_latest_dir_name(
str(self.prepare_options.backup_options.get("full_dir")))
# I am not going to initialize this object in Prepare class constructor as I thin there is no need.
backup_builder = BackupBuilderChecker(self.conf, dry_run=self.dry)
if found_backups[1] == "Full": # type: ignore
if recent_bck:
logger.info("- - - - Preparing Full Backup - - - -")
# Extracting/decrypting from streamed backup and extra checks goes here.
backup_builder.extract_decrypt_from_stream_backup(
recent_full_bck=recent_bck)
# Prepare command
backup_prepare_cmd = self.prepare_options.prepare_command_builder(
full_backup=recent_bck)
self.run_prepare_command(
str(self.prepare_options.backup_options.get("full_dir")),
recent_bck,
backup_prepare_cmd,
)
elif found_backups[1] == "Inc": # type: ignore
if not os.listdir(
str(self.prepare_options.backup_options.get("inc_dir"))):
logger.info(
"- - - - You have no Incremental backups. So will prepare only latest Full backup - - - -"
)
self.prepare_only_full_backup()
else:
logger.info("- - - - You have Incremental backups. - - - -")
if self.prepare_only_full_backup():
logger.info("Preparing Incs: ")
list_of_dir = helpers.sorted_ls(
str(self.prepare_options.backup_options.get(
"inc_dir")))
# Find the index number inside all list for backup(which was found via tag)
index_num = list_of_dir.index(
found_backups[0]) # type: ignore
# Limit the iteration until this found backup
for dir_ in list_of_dir[:index_num + 1]:
apply_log_only = None
if dir_ != found_backups[0]: # type: ignore
logger.info(
"Preparing inc backups in sequence. inc backup dir/name is {}"
.format(dir_))
apply_log_only = True
else:
logger.info(
"Preparing last incremental backup, inc backup dir/name is {}"
.format(dir_))
# Extracting/decrypting from streamed backup and extra checks goes here
backup_builder.extract_decrypt_from_stream_backup(
recent_inc_bck=dir_, flag=True)
# Prepare command
backup_prepare_cmd = (
self.prepare_options.prepare_command_builder(
full_backup=recent_bck,
incremental=dir_,
apply_log_only=apply_log_only,
))
self.run_prepare_command(
str(
self.prepare_options.backup_options.get(
"inc_dir")),
dir_,
backup_prepare_cmd,
)
logger.info("- - - - The end of the Prepare Stage. - - - -")
return True
def all_backup(self) -> bool:
"""
This method at first checks full backup directory, if it is empty takes full backup.
If it is not empty then checks for full backup time.
If the recent full backup is taken 1 day ago, it takes full backup.
In any other conditions it takes incremental backup.
"""
# Workaround for circular import dependency error in Python
# Creating object from CheckEnv class
check_env_obj = CheckEnv(
self.conf,
full_dir=str(self.builder_obj.backup_options.get("full_dir")),
inc_dir=str(self.builder_obj.backup_options.get("inc_dir")),
)
assert check_env_obj.check_all_env() is True, "environment checks failed!"
if not helpers.get_latest_dir_name(
str(self.builder_obj.backup_options.get("full_dir"))
):
logger.info(
"- - - - You have no backups : Taking very first Full Backup! - - - -"
)
if self.mysql_cli.mysql_run_command("flush logs") and self.full_backup():
# Removing old inc backups
self.clean_inc_backup_dir()
elif self.last_full_backup_date():
logger.info(
"- - - - Your full backup is timeout : Taking new Full Backup! - - - -"
)
# Archiving backups
if self.archive_obj.backup_archive_options.get("archive_dir"):
logger.info(
"Archiving enabled; cleaning archive_dir & archiving previous Full Backup"
)
if self.archive_obj.backup_archive_options.get(
"archive_max_duration"
) or self.archive_obj.backup_archive_options.get("archive_max_size"):
self.archive_obj.clean_old_archives()
self.archive_obj.create_backup_archives()
else:
logger.info("Archiving disabled. Skipping!")
if self.mysql_cli.mysql_run_command("flush logs") and self.full_backup():
# Removing full backups
self.clean_full_backup_dir()
# Removing inc backups
self.clean_inc_backup_dir()
else:
logger.info(
"- - - - You have a full backup that is less than {} seconds old. - - - -".format(
self.builder_obj.backup_options.get("full_backup_interval")
)
)
logger.info(
"- - - - We will take an incremental one based on recent Full Backup - - - -"
)
time.sleep(3)
# Taking incremental backup
self.inc_backup()
return True
def inc_backup(self) -> bool:
"""
Method for taking incremental backups.
:return: True on success.
:raise: RuntimeError on error.
"""
# Get the recent full backup path
recent_full_bck = helpers.get_latest_dir_name(
str(self.builder_obj.backup_options.get("full_dir"))
)
if not recent_full_bck:
raise RuntimeError(
"Failed to get Full backup path. Are you sure you have one?"
)
# Get the recent incremental backup path
recent_inc_bck = helpers.get_latest_dir_name(
str(self.builder_obj.backup_options.get("inc_dir"))
)
# Creating time-stamped incremental backup directory
inc_backup_dir = helpers.create_backup_directory(
str(self.builder_obj.backup_options.get("inc_dir"))
)
# Check here if stream=tar enabled.
# Because it is impossible to take incremental backup with streaming tar.
# raise RuntimeError.
self.builder_obj.stream_tar_incremental_checker()
xtrabackup_inc_cmd = self.builder_obj.inc_backup_command_builder(
recent_full_bck=recent_full_bck,
inc_backup_dir=inc_backup_dir,
recent_inc_bck=recent_inc_bck,
)
self.builder_obj.extract_decrypt_from_stream_backup(
recent_full_bck=recent_full_bck, recent_inc_bck=recent_inc_bck
)
# Deprecated workaround for LP #1444255
self.builder_obj.decrypter(
recent_full_bck=recent_full_bck,
xtrabackup_inc_cmd=xtrabackup_inc_cmd,
recent_inc_bck=recent_inc_bck,
)
if self.dry:
# If it's a dry run, skip running & tagging
return True
logger.debug(
"Starting {}".format(self.builder_obj.backup_options.get("backup_tool"))
)
status = ProcessRunner.run_command(xtrabackup_inc_cmd)
status_str = "OK" if status is True else "FAILED"
self.add_tag(
backup_type="Inc",
backup_size=helpers.get_folder_size(inc_backup_dir),
backup_status=status_str,
)
return status
