def is_success_mysql_xtrabackup(extra_keys, str_auth, backup_full_path, gzip, job_name): date_now = general_function.get_time_now('backup') tmp_status_file = f'/tmp/xtrabackup_status/{date_now}.log' dom = int(general_function.get_time_now('dom')) if dom == 1: dir_for_status_file = os.path.dirname(tmp_status_file) if os.path.isdir(dir_for_status_file): listing = glob.glob(dir_for_status_file) periodic_backup.delete_oldest_files(listing, 31, job_name) general_function.create_files(job_name, tmp_status_file) if gzip: dump_cmd = f"innobackupex {str_auth} {extra_keys} 2>{tmp_status_file} | gzip > {backup_full_path}" else: dump_cmd = f"innobackupex {str_auth} {extra_keys} > {backup_full_path} 2>{tmp_status_file} " command = general_function.exec_cmd(dump_cmd) code = command['code'] if not is_success_status_xtrabackup(tmp_status_file, job_name): log_and_mail.writelog( 'ERROR', f"Can't create xtrabackup in tmp directory! More information in status file {tmp_status_file}.", config.filelog_fd, job_name) return False elif code != 0: log_and_mail.writelog('ERROR', f"Bad result code external process '{dump_cmd}':'{code}'", config.filelog_fd, job_name) return False else: log_and_mail.writelog('INFO', "Successfully created xtrabackup in tmp directory.", config.filelog_fd, job_name) return True
def is_time_to_backup(job_data): """ A function that determines whether or not to run copy collection according to the plan. It receives a dictionary with data for a particular section at the input. """ job_name = job_data['job'] job_type = job_data['type'] storages = job_data['storages'] result = True if job_type == 'inc_files': return result dow = general_function.get_time_now("dow") dom = general_function.get_time_now("dom") day_flag = False week_flag = False month_flag = False for i in range(len(storages)): if storages[i]['enable']: if storages[i]['store']['days'] or storages[i]['store'][ 'weeks'] or storages[i]['store']['month']: if int(storages[i]['store']['days']) > 0: day_flag = True if int(storages[i]['store']['weeks']) > 0: week_flag = True if int(storages[i]['store']['month']) > 0: month_flag = True else: log_and_mail.writelog( 'ERROR', f'There are no stores data for storage {job_type} in the job {job_name}!', config.filelog_fd, job_name) continue if not day_flag: if not week_flag: if not month_flag: result = False else: if dom == config.dom_backup: result = True else: result = False else: if dow == config.dow_backup: result = True else: if not month_flag: result = False else: if dom == config.dom_backup: result = True else: result = True return result
def is_time_to_backup(job_data): ''''' Фукнция, которая определяет необходимо ли запускать на выполнение сбор копий согласно плану. На вход получает словарь с данными для конкретной секции ''' '' job_name = job_data['job'] job_type = job_data['type'] storages = job_data['storages'] if job_type == 'inc_files': return True dow = general_function.get_time_now("dow") dom = general_function.get_time_now("dom") day_flag = False week_flag = False month_flag = False for i in range(len(storages)): if storages[i]['enable']: if storages[i]['store']['days'] or storages[i]['store'][ 'weeks'] or storages[i]['store']['month']: if int(storages[i]['store']['days']) > 0: day_flag = True if int(storages[i]['store']['weeks']) > 0: week_flag = True if int(storages[i]['store']['month']) > 0: month_flag = True else: log_and_mail.writelog( 'ERROR', f'There are no stores data for storage {job_type} in the job {job_name}!', config.filelog_fd, job_name) continue if not day_flag: if not week_flag: if not month_flag: result = False else: if dom == config.dom_backup: result = True else: result = False else: if dow == config.dow_backup: result = True else: if not month_flag: result = False else: if dom == config.dom_backup: result = True else: result = True return result
def is_success_mysql_xtrabackup(extra_keys, str_auth, backup_full_path, gzip, job_name): date_now = general_function.get_time_now('backup') tmp_status_file = '/tmp/xtrabackup_status/%s.log' % (date_now) dom = int(general_function.get_time_now('dom')) if dom == 1: dir_for_status_file = os.path.dirname(tmp_status_file) if os.path.isdir(dir_for_status_file): listing = glob.glob(dir_for_status_file) periodic_backup.delete_oldest_files(listing, 31, job_name) general_function.create_files(job_name, tmp_status_file) if gzip: dump_cmd = "innobackupex %s %s 2>%s | gzip > %s" % ( str_auth, extra_keys, tmp_status_file, backup_full_path) else: dump_cmd = "innobackupex %s %s > %s 2>%s " % ( str_auth, extra_keys, backup_full_path, tmp_status_file) command = general_function.exec_cmd(dump_cmd) code = command['code'] if code != 0: log_and_mail.writelog( 'ERROR', "Bad result code external process '%s':'%s'" % (dump_cmd, code), config.filelog_fd, job_name) return False if not is_success_status_xtrabackup(tmp_status_file, job_name): log_and_mail.writelog( 'ERROR', "Can't create xtrabackup in tmp directory! More information in status file %s." % (tmp_status_file), config.filelog_fd, job_name) return False elif code != 0: log_and_mail.writelog( 'ERROR', "Bad result code external process '%s':'%s'" % (dump_cmd, code), config.filelog_fd, job_name) return False else: log_and_mail.writelog( 'INFO', "Successfully created xtrabackup in tmp directory.", config.filelog_fd, job_name) return True
def del_old_inc_file(old_year_dir, old_month_dirs): """ :param str old_year_dir: :param list old_month_dirs: """ for old_month_dir in old_month_dirs: general_function.del_file_objects('inc_files', old_month_dir) if os.path.isdir(old_year_dir): list_subdir_in_old_dir = os.listdir(old_year_dir) if len(list_subdir_in_old_dir) == 1 and \ list_subdir_in_old_dir[0] == 'year' and \ old_year_dir != general_function.get_time_now('year'): general_function.del_file_objects('inc_files', old_year_dir)
def get_log(log_level, log_message, type_message=''): """ The function of forming a string for writing to a log file. The input is given the following values: log_level - event level (error, info, warning); log_message - message; type_message is the section in the configuration file to which the event belongs. """ time_now = general_function.get_time_now('log') if type_message: result_str = f"{log_level} [{type_message}] [{time_now}]: {log_message}\n" else: result_str = f"{log_level} [{time_now}]: {log_message}\n" return result_str
def get_log(log_level, log_message, type_message=''): ''' The function of forming a string for writing to a log file. The input is given the following values: log_level - event level (error, info, warning); log_message - message; type_message is the section in the configuration file to which the event belongs. ''' time_now = general_function.get_time_now('log') if type_message: result_str = "%s [%s] [%s]: %s\n" % (log_level, type_message, time_now, log_message) else: result_str = "%s [%s]: %s\n" % (log_level, time_now, log_message) return result_str
def control_old_files(full_dir_path, store_backup_count, storage, job_name, host='', full_path_for_log='', share=''): dow = general_function.get_time_now("dow") dom = general_function.get_time_now("dom") files_grabbed_list = [] for extension in config.backup_extenstion: full_glob_path = os.path.join(full_dir_path, extension) files_grabbed_list.extend(glob.glob(full_glob_path)) count_file = len(files_grabbed_list) time_period = os.path.split(full_dir_path)[1] if int(store_backup_count): delta_count_file = int(count_file) - int(store_backup_count) if ((time_period == 'weekly' and dow != config.dow_backup) or (time_period == 'monthly' and dom != config.dom_backup)): result_delete_count = delta_count_file else: result_delete_count = delta_count_file + 1 if result_delete_count < 1: return 1 try: delete_oldest_files(files_grabbed_list, result_delete_count, job_name) except general_function.MyError as err: if storage == 'local': log_and_mail.writelog( 'ERROR', "Can't delete old '%s' files in directory '%s' on '%s' storage:%s" % (time_period, full_dir_path, storage, err), config.filelog_fd, job_name) elif storage == 'smb': log_and_mail.writelog( 'ERROR', "Can't delete old '%s' files in directory '%s' in '%s' share on '%s' storage(%s):%s" % (time_period, full_path_for_log, share, storage, host, err), config.filelog_fd, job_name) else: log_and_mail.writelog( 'ERROR', "Can't delete old '%s' files in directory '%s' on '%s' storage(%s):%s" % (time_period, full_path_for_log, storage, host, err), config.filelog_fd, job_name) else: if storage == 'local': log_and_mail.writelog( 'INFO', "Successfully deleted old '%s' files in directory '%s' on '%s' storage." % (time_period, full_dir_path, storage), config.filelog_fd, job_name) elif storage == 'smb': log_and_mail.writelog( 'INFO', "Successfully deleted old '%s' files in directory '%s' in '%s' share on '%s' storage(%s)." % (time_period, full_path_for_log, share, storage, host), config.filelog_fd, job_name) else: log_and_mail.writelog( 'INFO', "Successfully deleted old '%s' files in directory '%s' on '%s' storage(%s)." % (time_period, full_path_for_log, storage, host), config.filelog_fd, job_name) else: try: for i in files_grabbed_list: general_function.del_file_objects(job_name, i) except general_function.MyError as err: if storage == 'local': log_and_mail.writelog( 'ERROR', "Can't delete old '%s' files in directory '%s' on '%s' storage:%s" % (time_period, full_dir_path, storage, err), config.filelog_fd, job_name) elif storage == 'smb': log_and_mail.writelog( 'ERROR', "Can't delete old '%s' files in directory '%s' in '%s' share on '%s' storage(%s):%s" % (time_period, full_path_for_log, share, storage, host, err), config.filelog_fd, job_name) else: log_and_mail.writelog( 'ERROR', "Can't delete old '%s' files in directory '%s' on '%s' storage(%s):%s" % (time_period, full_path_for_log, storage, host, err), config.filelog_fd, job_name) else: if storage == 'local': log_and_mail.writelog( 'INFO', "Successfully deleted old '%s' files in directory '%s' on '%s' storage." % (time_period, full_dir_path, storage), config.filelog_fd, job_name) elif storage == 'smb': log_and_mail.writelog( 'INFO', "Successfully deleted old '%s' files in directory '%s' in '%s' share on '%s' storage(%s)." % (time_period, full_path_for_log, share, storage, host), config.filelog_fd, job_name) else: log_and_mail.writelog( 'INFO', "Successfully deleted old '%s' files in directory '%s' on '%s' storage(%s)." % (time_period, full_path_for_log, storage, host), config.filelog_fd, job_name)
def periodic_backup(full_tmp_path, general_local_dst_path, remote_dir, storage, subdir_name, days_count, weeks_count, job_name, host, share): daily_subdir_name = "daily" weekly_subdir_name = "weekly" monthly_subdir_name = "monthly" link_dict = {} dow = general_function.get_time_now("dow") backup_file_name = os.path.basename(full_tmp_path) full_dst_path = os.path.join(general_local_dst_path, subdir_name) dst_dirs = [] daily_dir = os.path.join(general_local_dst_path, daily_subdir_name) weekly_dir = os.path.join(general_local_dst_path, weekly_subdir_name) monthly_dir = os.path.join(general_local_dst_path, monthly_subdir_name) if storage == 'local': if subdir_name == monthly_subdir_name: dst_dirs.append(monthly_dir) if dow == config.dow_backup and int(weeks_count): src_link = os.path.join(general_local_dst_path, monthly_subdir_name, backup_file_name) dst_link = os.path.join(general_local_dst_path, weekly_subdir_name, backup_file_name) dst_dirs.append(weekly_dir) link_dict[dst_link] = src_link if int(days_count): src_link = os.path.join(general_local_dst_path, monthly_subdir_name, backup_file_name) dst_link = os.path.join(general_local_dst_path, daily_subdir_name, backup_file_name) dst_dirs.append(daily_dir) link_dict[dst_link] = src_link elif subdir_name == weekly_subdir_name and storage == 'local': dst_dirs.append(weekly_dir) if int(days_count): src_link = os.path.join(general_local_dst_path, weekly_subdir_name, backup_file_name) dst_link = os.path.join(general_local_dst_path, daily_subdir_name, backup_file_name) dst_dirs.append(daily_dir) link_dict[dst_link] = src_link else: dst_dirs.append(daily_dir) else: dst_dirs.append(full_dst_path) for dst_dir in set(dst_dirs): dirs_for_log = general_function.get_dirs_for_log( dst_dir, remote_dir, storage) general_function.create_dirs(job_name='', dirs_pairs={dst_dir: dirs_for_log}) if storage == 'local': try: general_function.move_ofs(full_tmp_path, full_dst_path) except general_function.MyError as err: log_and_mail.writelog( 'ERROR', "Can't move '%s' file '%s' -> '%s' on '%s' storage: %s" % (subdir_name, full_tmp_path, full_dst_path, storage, err), config.filelog_fd, job_name) else: log_and_mail.writelog( 'INFO', "Successfully moved '%s' file '%s' -> '%s' on '%s' storage." % (subdir_name, full_tmp_path, full_dst_path, storage), config.filelog_fd, job_name) if link_dict: for key in link_dict.keys(): src = link_dict[key] dst = key try: general_function.create_symlink(src, dst) except general_function.MyError as err: log_and_mail.writelog( 'ERROR', "Can't create symlink '%s' -> '%s' on 'local' storage: %s" % (src, dst, err), config.filelog_fd, job_name) else: try: general_function.copy_ofs(full_tmp_path, full_dst_path) dirs_for_log = general_function.get_dirs_for_log( full_dst_path, remote_dir, storage) except general_function.MyError as err: if storage != 'smb': log_and_mail.writelog( 'ERROR', "Can't copy '%s' file '%s' -> '%s' directory on '%s' storage(%s): %s" % (subdir_name, full_tmp_path, dirs_for_log, storage, host, err), config.filelog_fd, job_name) else: log_and_mail.writelog( 'ERROR', "Can't copy '%s' file '%s' -> '%s' directory in '%s' share on '%s' storage(%s): %s" % (subdir_name, full_tmp_path, dirs_for_log, share, storage, host, err), config.filelog_fd, job_name) else: if storage != 'smb': log_and_mail.writelog( 'INFO', "Successfully copied '%s' file '%s' -> '%s' directory on '%s' storage(%s)." % (subdir_name, full_tmp_path, dirs_for_log, storage, host), config.filelog_fd, job_name) else: log_and_mail.writelog( 'INFO', "Successfully copied '%s' file '%s' -> '%s' directory in '%s' share on '%s' storage(%s)." % (subdir_name, full_tmp_path, dirs_for_log, share, storage, host), config.filelog_fd, job_name)
def general_desc_iteration(full_tmp_path, storages, part_of_dir_path, job_name): dow = general_function.get_time_now("dow") dom = general_function.get_time_now("dom") index_local_storage = -1 for i in range(len(storages)): if storages[i]['storage'] == 'local': index_local_storage = i break if index_local_storage != -1: storages += [storages.pop(index_local_storage)] for i in range(len(storages)): if specific_function.is_save_to_storage(job_name, storages[i]): try: current_storage_data = mount_fuse.get_storage_data( job_name, storages[i]) except general_function.MyError as err: log_and_mail.writelog('ERROR', '%s' % (err), config.filelog_fd, job_name) continue else: storage = current_storage_data['storage'] backup_dir = current_storage_data['backup_dir'] try: mount_fuse.mount(current_storage_data) except general_function.MyError as err: log_and_mail.writelog( 'ERROR', "Can't mount remote '%s' storage :%s" % (storage, err), config.filelog_fd, job_name) continue else: remote_dir = '' # for logging if storage != 'local': remote_dir = backup_dir local_dst_dirname = mount_fuse.mount_point else: local_dst_dirname = backup_dir days_count = storages[i]['store']['days'] weeks_count = storages[i]['store']['weeks'] month_count = storages[i]['store']['month'] store_dict = { 'daily': days_count, 'weekly': weeks_count, 'monthly': month_count } if storage != 'local': if storage != 's3': host = current_storage_data['host'] else: host = '' if storage != 'smb': share = '' else: share = current_storage_data['share'] for j in list(store_dict.keys()): # For storage: sshfs, nfs backup_dir is the mount point and must already be created before mounting. # For storage: ftp, smb, webdav, s3 is NOT a mount point, but actually a relative path relative to the mount point if storage in ('scp', 'nfs'): full_path = os.path.join( local_dst_dirname, part_of_dir_path, j) remote_path_to_backup_dir = os.path.join( backup_dir, part_of_dir_path, j) else: full_path = os.path.join( local_dst_dirname, backup_dir.lstrip('/'), part_of_dir_path, j) remote_path_to_backup_dir = os.path.join( backup_dir.lstrip('/'), part_of_dir_path, j) store_backup_count = store_dict[j] control_old_files(full_path, store_backup_count, storage, job_name, host, remote_path_to_backup_dir, share) else: host = '' share = '' if int(month_count) and dom == config.dom_backup: subdir_name = 'monthly' elif int(weeks_count) and dow == config.dow_backup: subdir_name = 'weekly' elif int(days_count): subdir_name = 'daily' # For storage: sshfs, nfs backup_dir is the mount point and must already be created before mounting. # For storage: ftp, smb, webdav, s3 is NOT a mount point, but actually a relative path relative to the mount point if storage in ('local', 'scp', 'nfs'): general_local_dst_path = os.path.join( local_dst_dirname, part_of_dir_path) else: general_local_dst_path = os.path.join( local_dst_dirname, backup_dir.lstrip('/'), part_of_dir_path) periodic_backup(full_tmp_path, general_local_dst_path, remote_dir, storage, subdir_name, days_count, weeks_count, job_name, host, share) try: mount_fuse.unmount() except general_function.MyError as err: log_and_mail.writelog( 'ERROR', "Can't umount remote '%s' storage :%s" % (storage, err), config.filelog_fd, job_name) continue else: continue
def general_desc_iteration(full_tmp_path, storages, part_of_dir_path, job_name, safety_backup): dow = general_function.get_time_now("dow") dom = general_function.get_time_now("dom") index_local_storage = -1 for i in range(len(storages)): if storages[i]['storage'] == 'local': index_local_storage = i break if index_local_storage != -1: storages += [storages.pop(index_local_storage)] for i in range(len(storages)): if specific_function.is_save_to_storage(job_name, storages[i]): try: current_storage_data = mount_fuse.get_storage_data( job_name, storages[i]) except general_function.MyError as err: log_and_mail.writelog('ERROR', f'{err}', config.filelog_fd, job_name) continue else: storage = current_storage_data['storage'] backup_dir = current_storage_data['backup_dir'] try: mount_fuse.mount(current_storage_data) except general_function.MyError as err: log_and_mail.writelog( 'ERROR', f"Can't mount remote '{storage}' storage :{err}", config.filelog_fd, job_name) continue else: remote_dir = '' # for logging if storage != 'local': remote_dir = backup_dir local_dst_dirname = mount_fuse.mount_point + mount_fuse.mount_point_sub_dir else: local_dst_dirname = backup_dir days_count = storages[i]['store']['days'] weeks_count = storages[i]['store']['weeks'] month_count = storages[i]['store']['month'] store_dict = { 'daily': days_count, 'weekly': weeks_count, 'monthly': month_count } if storage != 'local': host, share = general_function.get_host_and_share( storage, current_storage_data) if not safety_backup: remove_old_remote_files(store_dict, storage, local_dst_dirname, part_of_dir_path, backup_dir, job_name, host, share, safety_backup) else: host = '' share = '' subdir_name = '' if int(month_count) and dom == config.dom_backup: subdir_name = 'monthly' elif int(weeks_count) and dow == config.dow_backup: subdir_name = 'weekly' elif int(days_count): subdir_name = 'daily' # For storage: sshfs, nfs backup_dir is the mount point and must already be created before mounting. # It'll be created if remote_mount_point is defined. # For storage: ftp, smb, webdav, s3 is NOT a mount point, but actually a relative path relative # to the mount point if storage in ('local', 'scp', 'nfs'): general_local_dst_path = os.path.join( local_dst_dirname, part_of_dir_path) else: general_local_dst_path = os.path.join( local_dst_dirname, backup_dir.lstrip('/'), part_of_dir_path) periodic_backup(full_tmp_path, general_local_dst_path, remote_dir, storage, subdir_name, days_count, weeks_count, job_name, host, share) if safety_backup and storage != 'local': remove_old_remote_files(store_dict, storage, local_dst_dirname, part_of_dir_path, backup_dir, job_name, host, share, safety_backup) try: mount_fuse.unmount() except general_function.MyError as err: log_and_mail.writelog( 'ERROR', f"Can't umount remote '{storage}' storage:{err}", config.filelog_fd, job_name) continue else: continue
def create_inc_file(local_dst_dirname, remote_dir, part_of_dir_path, backup_file_name, target, exclude_list, gzip, job_name, storage, host, share): ''' The function determines whether to collect a full backup or incremental, prepares all the necessary information. ''' date_year = general_function.get_time_now('year') date_month = general_function.get_time_now('moy') date_day = general_function.get_time_now('dom') if int(date_day) < 11: daily_prefix = 'day_01' elif int(date_day) < 21: daily_prefix = 'day_11' else: daily_prefix = 'day_21' year_dir = os.path.join(local_dst_dirname, part_of_dir_path, date_year) initial_dir = os.path.join(year_dir, 'year') # Path to full backup month_dir = os.path.join(year_dir, 'month_%s' % (date_month), 'monthly') daily_dir = os.path.join(year_dir, 'month_%s' % (date_month), 'daily', daily_prefix) year_inc_file = os.path.join(initial_dir, 'year.inc') month_inc_file = os.path.join(month_dir, 'month.inc') daily_inc_file = os.path.join(daily_dir, 'daily.inc') link_dict = {} # dict for symlink with pairs like dst: src copy_dict = {} # dict for copy with pairs like dst: src # Before we proceed to collect a copy, we need to delete the copies for the same month last year # if they are to not save extra archives old_year = int(date_year) - 1 old_year_dir = os.path.join(local_dst_dirname, part_of_dir_path, str(old_year)) if os.path.isdir(old_year_dir): old_month_dir = os.path.join(old_year_dir, 'month_%s' % (date_month)) del_old_inc_file(old_year_dir, old_month_dir) if not os.path.isfile(year_inc_file): # There is no original index file, so we need to check the existence of an year directory if os.path.isdir(year_dir): # There is a directory, but there is no file itself, then something went wrong, so # we delete this directory with all the data inside, because even if they are there # continue to collect incremental copies it will not be able to general_function.del_file_objects(job_name, year_dir) dirs_for_log = general_function.get_dirs_for_log( year_dir, remote_dir, storage) file_for_log = os.path.join(dirs_for_log, os.path.basename(year_inc_file)) log_and_mail.writelog('ERROR', "The file %s not found, so the directory %s is cleared." +\ "Incremental backup will be reinitialized " %(file_for_log, dirs_for_log), config.filelog_fd, job_name) # Initialize the incremental backup, i.e. collect a full copy dirs_for_log = general_function.get_dirs_for_log( initial_dir, remote_dir, storage) general_function.create_dirs(job_name=job_name, dirs_pairs={initial_dir: dirs_for_log}) # Get the current list of files and write to the year inc file meta_info = get_index(target, exclude_list) with open(year_inc_file, "w") as index_file: json.dump(meta_info, index_file) full_backup_path = general_function.get_full_path( initial_dir, backup_file_name, 'tar', gzip) general_files_func.create_tar('files', full_backup_path, target, gzip, 'inc_files', job_name, remote_dir, storage, host, share) # After creating the full copy, you need to make the symlinks for the inc.file and # the most collected copy in the month directory of the current month # as well as in the decade directory if it's local, scp the repository and # copy inc.file for other types of repositories that do not support symlynk. month_dirs_for_log = general_function.get_dirs_for_log( month_dir, remote_dir, storage) daily_dirs_for_log = general_function.get_dirs_for_log( daily_dir, remote_dir, storage) general_function.create_dirs(job_name=job_name, dirs_pairs={ month_dir: month_dirs_for_log, daily_dir: daily_dirs_for_log }) if storage in 'local, scp': link_dict[month_inc_file] = year_inc_file link_dict[os.path.join( month_dir, os.path.basename(full_backup_path))] = full_backup_path link_dict[daily_inc_file] = year_inc_file link_dict[os.path.join( daily_dir, os.path.basename(full_backup_path))] = full_backup_path else: copy_dict[month_inc_file] = year_inc_file copy_dict[daily_inc_file] = year_inc_file else: symlink_dir = '' if int(date_day) == 1: # It is necessary to collect monthly incremental backup relative to the year copy old_meta_info = specific_function.parser_json(year_inc_file) new_meta_info = get_index(target, exclude_list) general_inc_backup_dir = month_dir # It is also necessary to make a symlink for inc files and backups to the directory with the first decade symlink_dir = daily_dir general_dirs_for_log = general_function.get_dirs_for_log( general_inc_backup_dir, remote_dir, storage) symlink_dirs_for_log = general_function.get_dirs_for_log( symlink_dir, remote_dir, storage) general_function.create_dirs(job_name=job_name, dirs_pairs={ general_inc_backup_dir: general_dirs_for_log, symlink_dir: symlink_dirs_for_log }) with open(month_inc_file, "w") as index_file: json.dump(new_meta_info, index_file) elif int(date_day) == 11 or int(date_day) == 21: # It is necessary to collect a ten-day incremental backup relative to a monthly copy try: old_meta_info = specific_function.parser_json(month_inc_file) except general_function.MyError as e: log_and_mail.writelog( 'ERROR', "Couldn't open old month meta info file '%s': %s!" % (month_inc_file, e), config.filelog_fd, job_name) return 2 new_meta_info = get_index(target, exclude_list) general_inc_backup_dir = daily_dir general_dirs_for_log = general_function.get_dirs_for_log( general_inc_backup_dir, remote_dir, storage) general_function.create_dirs( job_name=job_name, dirs_pairs={general_inc_backup_dir: general_dirs_for_log}) with open(daily_inc_file, "w") as index_file: json.dump(new_meta_info, index_file) else: # It is necessary to collect a normal daily incremental backup relative to a ten-day copy try: old_meta_info = specific_function.parser_json(daily_inc_file) except general_function.MyError as e: log_and_mail.writelog( 'ERROR', "Couldn't open old decade meta info file '%s': %s!" % (daily_inc_file, e), config.filelog_fd, job_name) return 2 new_meta_info = get_index(target, exclude_list) general_inc_backup_dir = daily_dir general_dirs_for_log = general_function.get_dirs_for_log( general_inc_backup_dir, remote_dir, storage) general_function.create_dirs( job_name=job_name, dirs_pairs={general_inc_backup_dir: general_dirs_for_log}) # Calculate the difference between the old and new file states diff_json = compute_diff(new_meta_info, old_meta_info) inc_backup_path = general_function.get_full_path( general_inc_backup_dir, backup_file_name, 'tar', gzip) # Define the list of files that need to be included in the archive target_change_list = diff_json['modify'] # Form GNU.dumpdir headers dict_directory = {} # Dict to store pairs like dir:GNU.dumpdir excludes = r'|'.join([ fnmatch.translate(x)[:-7] for x in general_files_func.EXCLUDE_FILES ]) or r'$.' for dir_name, dirs, files in os.walk(target): first_level_files = [] if re.match(excludes, dir_name): continue for file in files: if re.match(excludes, os.path.join(dir_name, file)): continue first_level_files.append(file) first_level_subdirs = dirs dict_directory[dir_name] = get_gnu_dumpdir_format( diff_json, dir_name, target, excludes, first_level_subdirs, first_level_files) create_inc_tar(inc_backup_path, remote_dir, dict_directory, target_change_list, gzip, job_name, storage, host, share) if symlink_dir: if storage in 'local, scp': link_dict[daily_inc_file] = month_inc_file else: copy_dict[daily_inc_file] = month_inc_file if link_dict: for key in link_dict.keys(): src = link_dict[key] dst = key try: general_function.create_symlink(src, dst) except general_function.MyError as err: log_and_mail.writelog( 'ERROR', "Can't create symlink %s -> %s: %s" % (src, dst, err), config.filelog_fd, job_name) if copy_dict: for key in copy_dict.keys(): src = copy_dict[key] dst = key try: general_function.copy_ofs(src, dst) except general_function.MyError as err: log_and_mail.writelog( 'ERROR', "Can't copy %s -> %s: %s" % (src, dst, err), config.filelog_fd, job_name)
def control_files(full_dir_path, store_backup_count, storage, job_name, files_type, host='', full_path_for_log='', share='', safety_backup=False): dow = general_function.get_time_now("dow") dom = general_function.get_time_now("dom") files_grabbed_list = [] for extension in config.backup_extenstion: full_glob_path = os.path.join(full_dir_path, extension) files_grabbed_list.extend(glob.glob(full_glob_path)) count_file = len(files_grabbed_list) time_period = os.path.split(full_dir_path)[1] if int(store_backup_count): delta_count_file = int(count_file) - int(store_backup_count) if ((time_period == 'weekly' and dow != config.dow_backup) or (time_period == 'monthly' and dom != config.dom_backup)): result_delete_count = delta_count_file else: result_delete_count = delta_count_file + 1 if safety_backup: result_delete_count -= 1 if result_delete_count < 1: return 1 try: delete_oldest_files(files_grabbed_list, result_delete_count, job_name) except general_function.MyError as err: if storage == 'local': log_and_mail.writelog( 'ERROR', f"Can't delete {files_type} '{time_period}' files in directory '{full_dir_path}' on '{storage}' " f"storage: {err}", config.filelog_fd, job_name) elif storage == 'smb': log_and_mail.writelog( 'ERROR', f"Can't delete {files_type} '{time_period}' files in directory '{full_path_for_log}' in '{share}' " f"share on '{storage}' storage({host}): {err}", config.filelog_fd, job_name) else: log_and_mail.writelog( 'ERROR', f"Can't delete {files_type} '{time_period}' files in directory '{full_path_for_log}' on '{storage}' " f"storage({host}): {err}", config.filelog_fd, job_name) else: if storage == 'local': log_and_mail.writelog( 'INFO', f"Successfully deleted {files_type} '{time_period}' files in directory '{full_dir_path}' on '{storage}' " f"storage.", config.filelog_fd, job_name) elif storage == 'smb': log_and_mail.writelog( 'INFO', f"Successfully deleted {files_type} '{time_period}' files in directory '{full_path_for_log}' in '{share}' " f"share on '{storage}' storage({host}).", config.filelog_fd, job_name) else: log_and_mail.writelog( 'INFO', f"Successfully deleted {files_type} '{time_period}' files in directory '{full_path_for_log}' on '{storage}' " f"storage({host}).", config.filelog_fd, job_name) else: try: for i in files_grabbed_list: general_function.del_file_objects(job_name, i) except general_function.MyError as err: if storage == 'local': log_and_mail.writelog( 'ERROR', f"Can't delete {files_type} '{time_period}' files in directory '{full_dir_path}' on '{storage}' " f"storage:{err}", config.filelog_fd, job_name) elif storage == 'smb': log_and_mail.writelog( 'ERROR', f"Can't delete {files_type} '{time_period}' files in directory '{full_path_for_log}' in '{share}' " f"share on '{storage}' storage({host}):{err}", config.filelog_fd, job_name) else: log_and_mail.writelog( 'ERROR', f"Can't delete {files_type} '{time_period}' files in directory '{full_path_for_log}' on '{storage}' " f"storage({host}):{err}", config.filelog_fd, job_name) else: if storage == 'local': log_and_mail.writelog( 'INFO', f"Successfully deleted {files_type} '{time_period}' files in directory '{full_dir_path}' on '{storage}' " f"storage.", config.filelog_fd, job_name) elif storage == 'smb': log_and_mail.writelog( 'INFO', f"Successfully deleted {files_type} '{time_period}' files in directory '{full_path_for_log}' in '{share}' " f"share on '{storage}' storage({host}).", config.filelog_fd, job_name) else: log_and_mail.writelog( 'INFO', f"Successfully deleted {files_type} '{time_period}' files in directory '{full_path_for_log}' on '{storage}' " f"storage({host}).", config.filelog_fd, job_name)
def create_inc_backup(local_dst_dirname, remote_dir, part_of_dir_path, backup_file_name, target, exclude_list, gzip, job_name, storage, host, share, months_to_store): """ The function determines whether to collect a full backup or incremental, prepares all the necessary information. """ date_year = general_function.get_time_now('year') date_month = general_function.get_time_now('moy') date_day = general_function.get_time_now('dom') dated_paths = get_dated_paths(local_dst_dirname, part_of_dir_path, date_year, date_month, date_day) # Before we proceed to collect a copy, we need to delete the copies for the same month last year # if they are to not save extra archives old_month_dirs = [] if os.path.isdir(dated_paths['old_year_dir']) or months_to_store < 12: if months_to_store < 12: int_date_month = int(date_month) last_month = int_date_month - months_to_store if last_month <= 0: m_range = list(range(last_month+12, 13)) m_range.extend(list(range(1, int_date_month))) else: m_range = list(range(last_month, int_date_month)) for i in range(1, 13): if i not in m_range: date = str(i).zfill(2) if i < int(date_month): year_to_cleanup = dated_paths['year_dir'] else: year_to_cleanup = dated_paths['old_year_dir'] old_month_dirs.append(os.path.join(year_to_cleanup, f'month_{date}')) else: old_month_dirs.append(os.path.join(dated_paths['old_year_dir'], f'month_{date_month}')) del_old_inc_file(dated_paths['old_year_dir'], old_month_dirs) link_dict = {} # dict for symlink with pairs like dst: src copy_dict = {} # dict for copy with pairs like dst: src # Get the current list of files new_meta_info = get_index(target, exclude_list) if not os.path.isfile(dated_paths['year_inc_file']): # There is no original index file, so we need to check the existence of an year directory if os.path.isdir(dated_paths['year_dir']): # There is a directory, but there is no file itself, then something went wrong, so # we delete this directory with all the data inside, because even if they are there # continue to collect incremental copies it will not be able to general_function.del_file_objects(job_name, dated_paths['year_dir']) dirs_for_log = general_function.get_dirs_for_log(dated_paths['year_dir'], remote_dir, storage) file_for_log = os.path.join(dirs_for_log, os.path.basename(dated_paths['year_inc_file'])) log_and_mail.writelog('ERROR', f"The file {file_for_log} not found, so the directory {dirs_for_log} is cleared. " f"Incremental backup will be reinitialized ", config.filelog_fd, job_name) # Initialize the incremental backup, i.e. collect a full copy remote_dir_for_logs = general_function.get_dirs_for_log(dated_paths['initial_dir'], remote_dir, storage) general_function.create_dirs(job_name=job_name, dirs_pairs={dated_paths['initial_dir']: remote_dir_for_logs}) write_meta_info(dated_paths['year_inc_file'], new_meta_info) full_backup_path = general_function.get_full_path(dated_paths['initial_dir'], backup_file_name, 'tar', gzip) general_files_func.create_tar('files', full_backup_path, target, gzip, 'inc_files', job_name, remote_dir, storage, host, share) daily_dirs_remote = general_function.get_dirs_for_log(dated_paths['daily_dir'], remote_dir, storage) month_dirs_remote = general_function.get_dirs_for_log(dated_paths['month_dir'], remote_dir, storage) general_function.create_dirs(job_name=job_name, dirs_pairs={dated_paths['daily_dir']: daily_dirs_remote, dated_paths['month_dir']: month_dirs_remote}) if storage in 'local': link_dict[dated_paths['month_inc_file']] = dated_paths['year_inc_file'] link_dict[os.path.join(dated_paths['month_dir'], os.path.basename(full_backup_path))] = full_backup_path link_dict[dated_paths['daily_inc_file']] = dated_paths['year_inc_file'] link_dict[os.path.join(dated_paths['daily_dir'], os.path.basename(full_backup_path))] = full_backup_path elif storage in 'scp, nfs': copy_dict[dated_paths['month_inc_file']] = dated_paths['year_inc_file'] link_dict[os.path.join(dated_paths['month_dir'], os.path.basename(full_backup_path))] = \ full_backup_path.replace(local_dst_dirname, remote_dir) copy_dict[dated_paths['daily_inc_file']] = dated_paths['year_inc_file'] link_dict[os.path.join(dated_paths['daily_dir'], os.path.basename(full_backup_path))] = \ full_backup_path.replace(local_dst_dirname, remote_dir) else: copy_dict[dated_paths['month_inc_file']] = dated_paths['year_inc_file'] copy_dict[os.path.join(dated_paths['month_dir'], os.path.basename(full_backup_path))] = full_backup_path copy_dict[dated_paths['daily_inc_file']] = dated_paths['year_inc_file'] copy_dict[os.path.join(dated_paths['daily_dir'], os.path.basename(full_backup_path))] = full_backup_path else: symlink_dir = '' meta_path = '' if int(date_day) == 1: meta_path = dated_paths['month_inc_file'] old_meta_path = dated_paths['year_inc_file'] general_inc_backup_dir = dated_paths['month_dir'] symlink_dir = dated_paths['daily_dir'] elif int(date_day) == 11 or int(date_day) == 21: meta_path = dated_paths['daily_inc_file'] old_meta_path = dated_paths['month_inc_file'] general_inc_backup_dir = dated_paths['daily_dir'] else: old_meta_path = dated_paths['daily_inc_file'] general_inc_backup_dir = dated_paths['daily_dir'] try: old_meta_info = specific_function.parser_json(old_meta_path) except general_function.MyError as e: log_and_mail.writelog('ERROR', f"Couldn't open old meta info file '{old_meta_path}': {e}!", config.filelog_fd, job_name) return 2 general_dirs_for_log = general_function.get_dirs_for_log(general_inc_backup_dir, remote_dir, storage) general_function.create_dirs(job_name=job_name, dirs_pairs={general_inc_backup_dir: general_dirs_for_log}) if meta_path: write_meta_info(meta_path, new_meta_info) # Calculate the difference between the old and new file states diff_json = compute_diff(new_meta_info, old_meta_info) # Define the list of files that need to be included in the archive target_change_list = diff_json['modify'] dict_directory = get_dict_directory(target, diff_json) inc_backup_path = general_function.get_full_path(general_inc_backup_dir, backup_file_name, 'tar', gzip) create_inc_tar( inc_backup_path, remote_dir, dict_directory, target_change_list, gzip, job_name, storage, host, share ) if symlink_dir: symlink_dirs_for_log = general_function.get_dirs_for_log(symlink_dir, remote_dir, storage) general_function.create_dirs(job_name=job_name, dirs_pairs={symlink_dir: symlink_dirs_for_log}) if storage in 'local': link_dict[dated_paths['daily_inc_file']] = dated_paths['month_inc_file'] elif storage in 'scp, nfs': copy_dict[dated_paths['daily_inc_file'].replace(local_dst_dirname, remote_dir)] = \ dated_paths['month_inc_file'].replace(local_dst_dirname, remote_dir) else: copy_dict[dated_paths['daily_inc_file']] = dated_paths['month_inc_file'] create_links_and_copies(link_dict, copy_dict, job_name)