def external_backup(job_data):
""" Function, creates a external backup.
At the entrance receives a dictionary with the data of the job.
"""
job_name = 'undefined'
try:
job_name = job_data['job']
backup_type = job_data['type']
dump_cmd = job_data['dump_cmd']
storages = job_data['storages']
except KeyError as e:
log_and_mail.writelog('ERROR', f"Missing required key:'{e}'!",
config.filelog_fd, job_name)
return
safety_backup = job_data.get('safety_backup', False)
skip_backup_rotate = job_data.get('skip_backup_rotate', False)
periodic_backup.remove_local_file(storages, '', job_name)
command = general_function.exec_cmd(dump_cmd)
stderr = command['stderr']
stdout = command['stdout']
code = command['code']
if code != 0:
log_and_mail.writelog('ERROR',
f"Bad result code external process '{dump_cmd}': '{code}' with next STDERR:\n"
f"'{stderr}'",
config.filelog_fd, job_name)
return
if skip_backup_rotate:
log_and_mail.writelog('INFO', f"Command '{dump_cmd}' finished success with the next result:\n{stdout}",
config.filelog_fd, job_name)
return
source_dict = get_value_from_stdout(stderr, stdout, job_name)
if source_dict is None:
return
full_tmp_path = source_dict['full_path']
basename = source_dict['basename']
extension = source_dict['extension']
gzip = source_dict['gzip']
new_name = os.path.basename(general_function.get_full_path('', basename, extension, gzip))
new_full_tmp_path = os.path.join(os.path.dirname(full_tmp_path), new_name)
general_function.move_ofs(full_tmp_path, new_full_tmp_path)
periodic_backup.general_desc_iteration(new_full_tmp_path, storages, '',
job_name, safety_backup)
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type, '/var/cache/davfs2/*')
def del_old_inc_file(old_year_dir, old_month_dir):
general_function.del_file_objects('inc_files', old_month_dir)
list_subdir_in_old_dir = os.listdir(old_year_dir)
if len(list_subdir_in_old_dir) == 1 and list_subdir_in_old_dir[0] == 'year':
general_function.del_file_objects('inc_files', old_year_dir)
def is_success_bgsave(str_auth, backup_full_tmp_path, gzip, job_name):
backup_full_tmp_path_tmp = backup_full_tmp_path.split('.gz')[0]
dump_cmd = f"redis-cli {str_auth} --rdb {backup_full_tmp_path_tmp}"
command = general_function.exec_cmd(dump_cmd)
stderr = command['stderr']
check_success_cmd = "echo $?"
check_command = general_function.exec_cmd(check_success_cmd)
stdout = check_command.get('stdout')
if stdout == 1:
log_and_mail.writelog('ERROR', f"Can't create redis database dump '{backup_full_tmp_path_tmp}' in tmp directory:{stderr}",
config.filelog_fd, job_name)
return False
else:
if gzip:
try:
general_files_func.gzip_file(backup_full_tmp_path_tmp, backup_full_tmp_path)
except general_function.MyError as stderr:
log_and_mail.writelog('ERROR', f"Can't gzip redis database dump '{backup_full_tmp_path_tmp}' in tmp directory:{stderr}.",
config.filelog_fd, job_name)
return False
else:
log_and_mail.writelog('INFO', f"Successfully created redis database dump '{backup_full_tmp_path}' in tmp directory.",
config.filelog_fd, job_name)
return True
finally:
general_function.del_file_objects(job_name, backup_full_tmp_path_tmp)
else:
log_and_mail.writelog('INFO', f"Successfully created redis database dump '{backup_full_tmp_path_tmp}' in tmp directory.",
config.filelog_fd, job_name)
return True
def mysql_xtradb_backup(job_data):
try:
job_name = job_data['job']
backup_type = job_data['type']
tmp_dir = job_data['tmp_dir']
sources = job_data['sources']
storages = job_data['storages']
except KeyError as e:
log_and_mail.writelog('ERROR', "Missing required key:'%s'!" % (e),
config.filelog_fd, job_name)
return 1
full_path_tmp_dir = general_function.get_tmp_dir(tmp_dir, backup_type)
for i in range(len(sources)):
try:
connect = sources[i]['connect']
gzip = sources[i]['gzip']
extra_keys = sources[i]['extra_keys']
except KeyError as e:
log_and_mail.writelog('ERROR', "Missing required key:'%s'!" % (e),
config.filelog_fd, job_name)
continue
db_user = connect.get('db_user')
db_password = connect.get('db_password')
path_to_conf = connect.get('path_to_conf')
if not (path_to_conf and db_user and db_password):
log_and_mail.writelog(
'ERROR',
"Can't find the authentication data, please fill the required fields",
config.filelog_fd, job_name)
continue
if not os.path.isfile(path_to_conf):
log_and_mail.writelog('ERROR',
"Configuration file '%s' not found!",
config.filelog_fd, job_name)
continue
str_auth = '--defaults-file=%s --user=%s --password=%s' % (
path_to_conf, db_user, db_password)
backup_full_tmp_path = general_function.get_full_path(
full_path_tmp_dir, 'xtrabackup', 'tar', gzip)
periodic_backup.remove_old_local_file(storages, '', job_name)
if is_success_mysql_xtrabackup(extra_keys, str_auth,
backup_full_tmp_path, gzip, job_name):
periodic_backup.general_desc_iteration(backup_full_tmp_path,
storages, '', job_name)
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type, full_path_tmp_dir,
'/var/cache/davfs2/*')
def external_backup(job_data):
''''' Function, creates a external backup.
At the entrance receives a dictionary with the data of the job.
''' ''
try:
job_name = job_data['job']
backup_type = job_data['type']
dump_cmd = job_data['dump_cmd']
storages = job_data['storages']
except KeyError as e:
log_and_mail.writelog('ERROR', "Missing required key:'%s'!" % (e),
config.filelog_fd, job_name)
return 1
periodic_backup.remove_old_local_file(storages, '', job_name)
command = general_function.exec_cmd(dump_cmd)
stderr = command['stderr']
stdout = command['stdout']
code = command['code']
if code != 0:
log_and_mail.writelog(
'ERROR',
"Bad result code external process '%s': %s'" % (dump_cmd, code),
config.filelog_fd, job_name)
return 1
source_dict = get_value_from_stdout(stderr, stdout, job_name)
if source_dict is None:
return 1
full_tmp_path = source_dict['full_path']
basename = source_dict['basename']
extension = source_dict['extension']
gzip = source_dict['gzip']
new_name = os.path.basename(
general_function.get_full_path('', basename, extension, gzip))
new_full_tmp_path = os.path.join(os.path.dirname(full_tmp_path), new_name)
general_function.move_ofs(full_tmp_path, new_full_tmp_path)
periodic_backup.general_desc_iteration(new_full_tmp_path, storages, '',
job_name)
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type, '/var/cache/davfs2/*')
def unmount(storage):
if storage != 'local' and mount_point:
umount_cmd = f"fusermount -uz {mount_point}"
umount = general_function.exec_cmd(umount_cmd)
stderr_umount = umount['stderr']
code = umount['code']
if stderr_umount:
raise general_function.MyError(stderr_umount)
elif code != 0:
raise general_function.MyError(
f"Bad result code external process '{umount_cmd}':'{code}'")
else:
general_function.del_file_objects('', mount_point)
return
def unmount():
if mount_point:
umount_cmd = "fusermount -uz %s" % (mount_point)
umount = general_function.exec_cmd(umount_cmd)
stderr_umount = umount['stderr']
code = umount['code']
if stderr_umount:
raise general_function.MyError(stderr_umount)
elif code != 0:
raise general_function.MyError(
"Bad result code external process '%s':'%s'" %
(umount_cmd, code))
else:
general_function.del_file_objects('', mount_point)
return 1
def del_old_inc_file(old_year_dir, old_month_dirs):
"""
:param str old_year_dir:
:param list old_month_dirs:
"""
for old_month_dir in old_month_dirs:
general_function.del_file_objects('inc_files', old_month_dir)
if os.path.isdir(old_year_dir):
list_subdir_in_old_dir = os.listdir(old_year_dir)
if len(list_subdir_in_old_dir) == 1 and \
list_subdir_in_old_dir[0] == 'year' and \
old_year_dir != general_function.get_time_now('year'):
general_function.del_file_objects('inc_files', old_year_dir)
def mysql_xtrabackup(job_data):
is_prams_read, job_name, backup_type, tmp_dir, sources, storages, safety_backup, deferred_copying_level = \
general_function.get_job_parameters(job_data)
if not is_prams_read:
return
full_path_tmp_dir = general_function.get_tmp_dir(tmp_dir, backup_type)
for i in range(len(sources)):
try:
connect = sources[i]['connect']
gzip = sources[i]['gzip']
extra_keys = sources[i]['extra_keys']
except KeyError as e:
log_and_mail.writelog('ERROR', f"Missing required key:'{e}'!", config.filelog_fd, job_name)
continue
db_user = connect.get('db_user')
db_password = connect.get('db_password')
path_to_conf = connect.get('path_to_conf')
if not (path_to_conf and db_user and db_password):
log_and_mail.writelog('ERROR', "Can't find the authentication data, please fill the required fields",
config.filelog_fd, job_name)
continue
if not os.path.isfile(path_to_conf):
log_and_mail.writelog('ERROR', f"Configuration file '{path_to_conf}' not found!",
config.filelog_fd, job_name)
continue
str_auth = f'--defaults-file={path_to_conf} --user={db_user} --password={db_password}'
backup_full_tmp_path = general_function.get_full_path(full_path_tmp_dir, 'xtrabackup', 'tar', gzip)
periodic_backup.remove_old_local_file(storages, '', job_name)
if is_success_mysql_xtrabackup(extra_keys, str_auth, backup_full_tmp_path, gzip, job_name):
periodic_backup.general_desc_iteration(backup_full_tmp_path, storages, '', job_name, safety_backup)
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type, full_path_tmp_dir, '/var/cache/davfs2/*')
def delete_oldest_files(files_list, count, job_name):
tmp_list = []
for i in files_list:
if os.path.exists(i):
tmp_list.append(i)
else:
general_function.del_file_objects(job_name, i)
count -= 1
time_sorted_list = sorted(tmp_list, key=os.path.getmtime)
length_list = len(time_sorted_list)
if count <= 0:
return 0
elif count > length_list:
count = length_list
for i in time_sorted_list[0:count]:
general_function.del_file_objects(job_name, i)
def mongodb_backup(job_data):
''' Function, creates a mongodb backup.
At the entrance receives a dictionary with the data of the job.
'''
try:
job_name = job_data['job']
backup_type = job_data['type']
tmp_dir = job_data['tmp_dir']
sources = job_data['sources']
storages = job_data['storages']
except KeyError as e:
log_and_mail.writelog('ERROR', "Missing required key:'%s'!" %(e), config.filelog_fd, job_name)
return 1
full_path_tmp_dir = general_function.get_tmp_dir(tmp_dir, backup_type)
for i in range(len(sources)):
exclude_dbs_list = sources[i].get('exclude_dbs', [])
exclude_collections_list = sources[i].get('exclude_collections', [])
try:
connect = sources[i]['connect']
target_db_list = sources[i]['target_dbs']
target_collection_list = sources[i]['target_collections']
gzip = sources[i]['gzip']
extra_keys = sources[i]['extra_keys']
except KeyError as e:
log_and_mail.writelog('ERROR', "Missing required key:'%s'!" %(e), config.filelog_fd, job_name)
continue
db_host = connect.get('db_host')
db_port = connect.get('db_port')
db_user = connect.get('db_user')
db_password = connect.get('db_password')
if not (db_host and not (bool(db_user) ^ bool(db_password))):
log_and_mail.writelog('ERROR', "Can't find the authentication data, please fill in the required fields",
config.filelog_fd, job_name)
continue
if not db_port:
db_port = general_function.get_default_port('mongodb')
is_all_flag_db = is_all_flag_collection = False
if 'all' in target_db_list:
is_all_flag_db = True
if 'all' in target_collection_list:
is_all_flag_collection = True
if db_user:
uri = "mongodb://%s:%s@%s:%s/" % (db_user, db_password, db_host, db_port) # for pymongo
str_auth = " --host %s --port %s --username %s --password %s " %(db_host, db_port, db_user, db_password) # for mongodump
else:
uri = "mongodb://%s:%s/" % (db_host, db_port)
str_auth = " --host %s --port %s " %(db_host, db_port)
if is_all_flag_db:
try:
client = pymongo.MongoClient(uri)
target_db_list = client.database_names()
except pymongo.errors.PyMongoError as err:
log_and_mail.writelog('ERROR', "Can't connect to MongoDB instances with the following data host='%s', port='%s', user='******', passwd='%s':%s" %(db_host, db_port, db_user, db_password, err),
config.filelog_fd, job_name)
continue
finally:
client.close()
for db in target_db_list:
if not db in exclude_dbs_list:
try:
client = pymongo.MongoClient(uri)
current_db = client[db]
collection_list = current_db.collection_names()
except pymongo.errors.PyMongoError as err:
log_and_mail.writelog('ERROR', "Can't connect to MongoDB instances with the following data host='%s', port='%s', user='******', passwd='%s':%s" %(db_host, db_port, db_user, db_password, err),
config.filelog_fd, job_name)
continue
finally:
client.close()
if is_all_flag_collection:
target_collection_list = collection_list
for collection in target_collection_list:
if not collection in exclude_collections_list and collection in collection_list:
str_auth_finally = "%s --collection %s " %(str_auth, collection)
backup_full_tmp_path = general_function.get_full_path(
full_path_tmp_dir,
collection,
'mongodump',
gzip)
part_of_dir_path = os.path.join(db, collection)
periodic_backup.remove_old_local_file(storages, part_of_dir_path, job_name)
if is_success_mongodump(collection, db, extra_keys, str_auth_finally, backup_full_tmp_path, gzip, job_name):
periodic_backup.general_desc_iteration(backup_full_tmp_path,
storages, part_of_dir_path,
job_name)
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type,
full_path_tmp_dir, '/var/cache/davfs2/*')
def postgresql_basebackup(job_data):
is_prams_read, job_name, backup_type, tmp_dir, sources, storages, safety_backup, deferred_copying_level = \
general_function.get_job_parameters(job_data)
if not is_prams_read:
return
full_path_tmp_dir = general_function.get_tmp_dir(tmp_dir, backup_type)
for i in range(len(sources)):
try:
connect = sources[i]['connect']
gzip = sources[i]['gzip']
extra_keys = sources[i]['extra_keys']
except KeyError as e:
log_and_mail.writelog('ERROR', f"Missing required key:'{e}'!",
config.filelog_fd, job_name)
continue
db_host = connect.get('db_host')
db_port = connect.get('db_port')
db_user = connect.get('db_user')
db_password = connect.get('db_password')
if not (db_user or db_host or db_password):
log_and_mail.writelog(
'ERROR',
"Can't find the authentication data, please fill in the required fields",
config.filelog_fd, job_name)
continue
if not db_port:
db_port = general_function.get_default_port('postgresql')
try:
connection = psycopg2.connect(dbname="postgres",
user=db_user,
password=db_password,
host=db_host,
port=db_port)
except psycopg2.Error as err:
log_and_mail.writelog(
'ERROR',
f"Can't connect to PostgreSQL instances with with following data host='{db_host}', "
f"port='{db_port}', user='******', passwd='{db_password}':{err}",
config.filelog_fd, job_name)
continue
else:
connection.close()
backup_full_tmp_path = general_function.get_full_path(
full_path_tmp_dir, 'postgresq_hot', 'tar', gzip, i)
periodic_backup.remove_old_local_file(storages, '', job_name)
str_auth = f' --dbname=postgresql://{db_user}:{db_password}@{db_host}:{db_port}/ '
if is_success_pgbasebackup(extra_keys, str_auth, backup_full_tmp_path,
gzip, job_name):
periodic_backup.general_desc_iteration(backup_full_tmp_path,
storages, '', job_name,
safety_backup)
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type, full_path_tmp_dir,
'/var/cache/davfs2/*')
def mongodb_backup(job_data):
""" Function, creates a mongodb backup.
At the entrance receives a dictionary with the data of the job.
"""
is_prams_read, job_name, backup_type, tmp_dir, sources, storages, safety_backup, deferred_copying_level = \
general_function.get_job_parameters(job_data)
if not is_prams_read:
return
full_path_tmp_dir = general_function.get_tmp_dir(tmp_dir, backup_type)
dumped_collections = {}
for i in range(len(sources)):
exclude_dbs_list = sources[i].get('exclude_dbs', [])
exclude_collections_list = sources[i].get('exclude_collections', [])
try:
connect = sources[i]['connect']
target_db_list = sources[i]['target_dbs']
target_collection_list = sources[i]['target_collections']
gzip = sources[i]['gzip']
extra_keys = sources[i]['extra_keys']
except KeyError as e:
log_and_mail.writelog('ERROR', f"Missing required key:'{e}'!", config.filelog_fd, job_name)
continue
db_host = connect.get('db_host')
db_port = connect.get('db_port')
db_user = connect.get('db_user')
db_password = connect.get('db_password')
if not (db_host and not (bool(db_user) ^ bool(db_password))):
log_and_mail.writelog('ERROR', "Can't find the authentication data, please fill in the required fields",
config.filelog_fd, job_name)
continue
if not db_port:
db_port = general_function.get_default_port('mongodb')
is_all_flag_db = is_all_flag_collection = False
if 'all' in target_db_list:
is_all_flag_db = True
if 'all' in target_collection_list:
is_all_flag_collection = True
if db_user:
uri = f"mongodb://{db_user}:{db_password}@{db_host}:{db_port}/" # for pymongo
str_auth = f" --host {db_host} --port {db_port} --username {db_user} --password {db_password} "
else:
uri = f"mongodb://{db_host}:{db_port}/"
str_auth = f" --host {db_host} --port {db_port} "
client = None
if is_all_flag_db:
try:
client = pymongo.MongoClient(uri)
target_db_list = client.list_database_names()
except PyMongoError as err:
log_and_mail.writelog('ERROR',
f"Can't connect to MongoDB instances with the following data host='{db_host}', "
f"port='{db_port}', user='******', passwd='{db_password}':{err}",
config.filelog_fd, job_name)
continue
finally:
if client:
client.close()
for db in target_db_list:
if db not in exclude_dbs_list:
try:
client = pymongo.MongoClient(uri)
current_db = client[db]
collection_list = current_db.collection_names()
except PyMongoError as err:
log_and_mail.writelog(
'ERROR',
f"Can't connect to MongoDB instances with the following data host='{db_host}', "
f"port='{db_port}', user='******', passwd='{db_password}':{err}", config.filelog_fd,
job_name)
continue
finally:
if client:
client.close()
if is_all_flag_collection:
target_collection_list = collection_list
for collection in target_collection_list:
if collection not in exclude_collections_list and collection in collection_list:
str_auth_finally = f"{str_auth} --collection {collection} "
backup_full_tmp_path = general_function.get_full_path(
full_path_tmp_dir,
collection,
'mongodump',
gzip,
f'{i}-{db}-')
part_of_dir_path = os.path.join(db, collection)
periodic_backup.remove_old_local_file(storages, part_of_dir_path, job_name)
if is_success_mongodump(collection, db, extra_keys, str_auth_finally, backup_full_tmp_path,
gzip, job_name):
dumped_collections[collection] = {'success': True,
'tmp_path': backup_full_tmp_path,
'part_of_dir_path': part_of_dir_path}
else:
dumped_collections[collection] = {'success': False}
if deferred_copying_level <= 0 and dumped_collections[collection]['success']:
periodic_backup.general_desc_iteration(backup_full_tmp_path,
storages, part_of_dir_path,
job_name, safety_backup)
for collection, result in dumped_collections.items():
if deferred_copying_level == 1 and result['success']:
periodic_backup.general_desc_iteration(result['tmp_path'], storages,
result['part_of_dir_path'], job_name, safety_backup)
for collection, result in dumped_collections.items():
if deferred_copying_level == 2 and result['success']:
periodic_backup.general_desc_iteration(result['tmp_path'], storages,
result['part_of_dir_path'], job_name, safety_backup)
for collection, result in dumped_collections.items():
if deferred_copying_level >= 3 and result['success']:
periodic_backup.general_desc_iteration(result['tmp_path'], storages,
result['part_of_dir_path'], job_name, safety_backup)
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type, full_path_tmp_dir, '/var/cache/davfs2/*')
def redis_backup(job_data):
try:
job_name = job_data['job']
backup_type = job_data['type']
tmp_dir = job_data['tmp_dir']
sources = job_data['sources']
storages = job_data['storages']
except KeyError as e:
log_and_mail.writelog('ERROR', "Missing required key:'%s'!" %(e), config.filelog_fd, job_name)
return 1
full_path_tmp_dir = general_function.get_tmp_dir(tmp_dir, backup_type)
for i in range(len(sources)):
try:
connect = sources[i]['connect']
gzip = sources[i]['gzip']
except KeyError as e:
log_and_mail.writelog('ERROR', "Missing required key:'%s'!" %(e), config.filelog_fd, job_name)
continue
db_host = connect.get('db_host')
db_port = connect.get('db_port')
db_password = connect.get('db_password')
socket = connect.get('socket')
if not (db_host or socket):
log_and_mail.writelog('ERROR', "Can't find the authentication data, please fill in the required fields",
config.filelog_fd, job_name)
continue
if not db_port:
db_port = general_function.get_default_port('redis')
try:
if db_host:
if db_password:
redis.StrictRedis(host=db_host, port=db_port, password=db_password)
str_auth = " -h %s -p %s -a '%s' " %(db_host, db_port, db_password)
else:
redis.StrictRedis(host=db_host, port=db_port)
str_auth = " -h %s -p %s " %(db_host, db_port)
else:
if db_password:
redis.StrictRedis(unix_socket_path=socket, password=db_password)
str_auth = " -s %s -a '%s' " %(socket, db_password)
else:
redis.StrictRedis(unix_socket_path=socket)
str_auth = " -s %s " %(socket)
except (redis.exceptions.ConnectionError, ConnectionRefusedError) as err:
log_and_mail.writelog('ERROR', "Can't connect to Redis instances with with following data host='%s', port='%s', passwd='%s', socket='%s' :%s" %(db_host, db_port, db_password, socket, err),
config.filelog_fd, job_name)
continue
else:
backup_full_tmp_path = general_function.get_full_path(
full_path_tmp_dir,
'redis',
'rdb',
gzip)
periodic_backup.remove_old_local_file(storages, '', job_name)
if is_success_bgsave(str_auth, backup_full_tmp_path, gzip, job_name):
periodic_backup.general_desc_iteration(backup_full_tmp_path,
storages, '',
job_name)
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type,
full_path_tmp_dir, '/var/cache/davfs2/*')
def create_inc_file(local_dst_dirname, remote_dir, part_of_dir_path,
backup_file_name, target, exclude_list, gzip, job_name,
storage, host, share):
''' The function determines whether to collect a full backup or incremental,
prepares all the necessary information.
'''
date_year = general_function.get_time_now('year')
date_month = general_function.get_time_now('moy')
date_day = general_function.get_time_now('dom')
if int(date_day) < 11:
daily_prefix = 'day_01'
elif int(date_day) < 21:
daily_prefix = 'day_11'
else:
daily_prefix = 'day_21'
year_dir = os.path.join(local_dst_dirname, part_of_dir_path, date_year)
initial_dir = os.path.join(year_dir, 'year') # Path to full backup
month_dir = os.path.join(year_dir, 'month_%s' % (date_month), 'monthly')
daily_dir = os.path.join(year_dir, 'month_%s' % (date_month), 'daily',
daily_prefix)
year_inc_file = os.path.join(initial_dir, 'year.inc')
month_inc_file = os.path.join(month_dir, 'month.inc')
daily_inc_file = os.path.join(daily_dir, 'daily.inc')
link_dict = {} # dict for symlink with pairs like dst: src
copy_dict = {} # dict for copy with pairs like dst: src
# Before we proceed to collect a copy, we need to delete the copies for the same month last year
# if they are to not save extra archives
old_year = int(date_year) - 1
old_year_dir = os.path.join(local_dst_dirname, part_of_dir_path,
str(old_year))
if os.path.isdir(old_year_dir):
old_month_dir = os.path.join(old_year_dir, 'month_%s' % (date_month))
del_old_inc_file(old_year_dir, old_month_dir)
if not os.path.isfile(year_inc_file):
# There is no original index file, so we need to check the existence of an year directory
if os.path.isdir(year_dir):
# There is a directory, but there is no file itself, then something went wrong, so
# we delete this directory with all the data inside, because even if they are there
# continue to collect incremental copies it will not be able to
general_function.del_file_objects(job_name, year_dir)
dirs_for_log = general_function.get_dirs_for_log(
year_dir, remote_dir, storage)
file_for_log = os.path.join(dirs_for_log,
os.path.basename(year_inc_file))
log_and_mail.writelog('ERROR', "The file %s not found, so the directory %s is cleared." +\
"Incremental backup will be reinitialized " %(file_for_log, dirs_for_log),
config.filelog_fd, job_name)
# Initialize the incremental backup, i.e. collect a full copy
dirs_for_log = general_function.get_dirs_for_log(
initial_dir, remote_dir, storage)
general_function.create_dirs(job_name=job_name,
dirs_pairs={initial_dir: dirs_for_log})
# Get the current list of files and write to the year inc file
meta_info = get_index(target, exclude_list)
with open(year_inc_file, "w") as index_file:
json.dump(meta_info, index_file)
full_backup_path = general_function.get_full_path(
initial_dir, backup_file_name, 'tar', gzip)
general_files_func.create_tar('files', full_backup_path, target, gzip,
'inc_files', job_name, remote_dir,
storage, host, share)
# After creating the full copy, you need to make the symlinks for the inc.file and
# the most collected copy in the month directory of the current month
# as well as in the decade directory if it's local, scp the repository and
# copy inc.file for other types of repositories that do not support symlynk.
month_dirs_for_log = general_function.get_dirs_for_log(
month_dir, remote_dir, storage)
daily_dirs_for_log = general_function.get_dirs_for_log(
daily_dir, remote_dir, storage)
general_function.create_dirs(job_name=job_name,
dirs_pairs={
month_dir: month_dirs_for_log,
daily_dir: daily_dirs_for_log
})
if storage in 'local, scp':
link_dict[month_inc_file] = year_inc_file
link_dict[os.path.join(
month_dir,
os.path.basename(full_backup_path))] = full_backup_path
link_dict[daily_inc_file] = year_inc_file
link_dict[os.path.join(
daily_dir,
os.path.basename(full_backup_path))] = full_backup_path
else:
copy_dict[month_inc_file] = year_inc_file
copy_dict[daily_inc_file] = year_inc_file
else:
symlink_dir = ''
if int(date_day) == 1:
# It is necessary to collect monthly incremental backup relative to the year copy
old_meta_info = specific_function.parser_json(year_inc_file)
new_meta_info = get_index(target, exclude_list)
general_inc_backup_dir = month_dir
# It is also necessary to make a symlink for inc files and backups to the directory with the first decade
symlink_dir = daily_dir
general_dirs_for_log = general_function.get_dirs_for_log(
general_inc_backup_dir, remote_dir, storage)
symlink_dirs_for_log = general_function.get_dirs_for_log(
symlink_dir, remote_dir, storage)
general_function.create_dirs(job_name=job_name,
dirs_pairs={
general_inc_backup_dir:
general_dirs_for_log,
symlink_dir: symlink_dirs_for_log
})
with open(month_inc_file, "w") as index_file:
json.dump(new_meta_info, index_file)
elif int(date_day) == 11 or int(date_day) == 21:
# It is necessary to collect a ten-day incremental backup relative to a monthly copy
try:
old_meta_info = specific_function.parser_json(month_inc_file)
except general_function.MyError as e:
log_and_mail.writelog(
'ERROR',
"Couldn't open old month meta info file '%s': %s!" %
(month_inc_file, e), config.filelog_fd, job_name)
return 2
new_meta_info = get_index(target, exclude_list)
general_inc_backup_dir = daily_dir
general_dirs_for_log = general_function.get_dirs_for_log(
general_inc_backup_dir, remote_dir, storage)
general_function.create_dirs(
job_name=job_name,
dirs_pairs={general_inc_backup_dir: general_dirs_for_log})
with open(daily_inc_file, "w") as index_file:
json.dump(new_meta_info, index_file)
else:
# It is necessary to collect a normal daily incremental backup relative to a ten-day copy
try:
old_meta_info = specific_function.parser_json(daily_inc_file)
except general_function.MyError as e:
log_and_mail.writelog(
'ERROR',
"Couldn't open old decade meta info file '%s': %s!" %
(daily_inc_file, e), config.filelog_fd, job_name)
return 2
new_meta_info = get_index(target, exclude_list)
general_inc_backup_dir = daily_dir
general_dirs_for_log = general_function.get_dirs_for_log(
general_inc_backup_dir, remote_dir, storage)
general_function.create_dirs(
job_name=job_name,
dirs_pairs={general_inc_backup_dir: general_dirs_for_log})
# Calculate the difference between the old and new file states
diff_json = compute_diff(new_meta_info, old_meta_info)
inc_backup_path = general_function.get_full_path(
general_inc_backup_dir, backup_file_name, 'tar', gzip)
# Define the list of files that need to be included in the archive
target_change_list = diff_json['modify']
# Form GNU.dumpdir headers
dict_directory = {} # Dict to store pairs like dir:GNU.dumpdir
excludes = r'|'.join([
fnmatch.translate(x)[:-7] for x in general_files_func.EXCLUDE_FILES
]) or r'$.'
for dir_name, dirs, files in os.walk(target):
first_level_files = []
if re.match(excludes, dir_name):
continue
for file in files:
if re.match(excludes, os.path.join(dir_name, file)):
continue
first_level_files.append(file)
first_level_subdirs = dirs
dict_directory[dir_name] = get_gnu_dumpdir_format(
diff_json, dir_name, target, excludes, first_level_subdirs,
first_level_files)
create_inc_tar(inc_backup_path, remote_dir, dict_directory,
target_change_list, gzip, job_name, storage, host,
share)
if symlink_dir:
if storage in 'local, scp':
link_dict[daily_inc_file] = month_inc_file
else:
copy_dict[daily_inc_file] = month_inc_file
if link_dict:
for key in link_dict.keys():
src = link_dict[key]
dst = key
try:
general_function.create_symlink(src, dst)
except general_function.MyError as err:
log_and_mail.writelog(
'ERROR',
"Can't create symlink %s -> %s: %s" % (src, dst, err),
config.filelog_fd, job_name)
if copy_dict:
for key in copy_dict.keys():
src = copy_dict[key]
dst = key
try:
general_function.copy_ofs(src, dst)
except general_function.MyError as err:
log_and_mail.writelog(
'ERROR', "Can't copy %s -> %s: %s" % (src, dst, err),
config.filelog_fd, job_name)
def mysql_backup(job_data):
try:
job_name = job_data['job']
backup_type = job_data['type']
tmp_dir = job_data['tmp_dir']
sources = job_data['sources']
storages = job_data['storages']
except KeyError as e:
log_and_mail.writelog('ERROR', f"Missing required key:'{e}'!",
config.filelog_fd, job_name)
return 1
full_path_tmp_dir = general_function.get_tmp_dir(tmp_dir, backup_type)
for i in range(len(sources)):
exclude_list = sources[i].get('excludes', [])
try:
connect = sources[i]['connect']
target_list = sources[i]['target']
gzip = sources[i]['gzip']
is_slave = sources[i]['is_slave']
extra_keys = sources[i]['extra_keys']
except KeyError as e:
log_and_mail.writelog('ERROR', f"Missing required key:'{e}'!",
config.filelog_fd, job_name)
continue
db_host = connect.get('db_host')
db_port = connect.get('db_port')
socket = connect.get('socket')
db_user = connect.get('db_user')
db_password = connect.get('db_password')
auth_file = connect.get('auth_file')
if not (auth_file or
((db_host or socket) and db_user and db_password)):
log_and_mail.writelog(
'ERROR',
"Can't find the authentication data, please fill in the required fields",
config.filelog_fd, job_name)
continue
if not db_port:
db_port = general_function.get_default_port('mysql')
is_all_flag = False
if 'all' in target_list:
is_all_flag = True
try:
(connection_1,
str_auth) = get_connection(db_host, db_port, db_user, db_password,
auth_file, socket, job_name)
except:
continue
cur_1 = connection_1.cursor()
if is_all_flag:
cur_1.execute("SHOW DATABASES")
target_list = [i[0] for i in cur_1.fetchall()]
if is_slave:
try:
cur_1.execute("STOP SLAVE")
except MySQLdb.Error as err:
log_and_mail.writelog('ERROR', f"Can't stop slave: {err}",
config.filelog_fd, job_name)
connection_1.close()
for db in target_list:
if not db in exclude_list:
backup_full_tmp_path = general_function.get_full_path(
full_path_tmp_dir, db, 'sql', gzip)
periodic_backup.remove_old_local_file(storages, db, job_name)
if is_success_mysqldump(db, extra_keys, str_auth,
backup_full_tmp_path, gzip, job_name):
periodic_backup.general_desc_iteration(
backup_full_tmp_path, storages, db, job_name)
if is_slave:
try:
(connection_2,
str_auth) = get_connection(db_host, db_port, db_user,
db_password, auth_file, socket,
job_name)
cur_2 = connection_2.cursor()
cur_2.execute("START SLAVE")
except MySQLdb.Error as err:
log_and_mail.writelog('ERROR', f"Can't start slave: {err} ",
config.filelog_fd, job_name)
finally:
connection_2.close()
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type, full_path_tmp_dir,
'/var/cache/davfs2/*')
def control_files(full_dir_path, store_backup_count, storage, job_name, files_type,
host='', full_path_for_log='', share='', safety_backup=False):
dow = general_function.get_time_now("dow")
dom = general_function.get_time_now("dom")
files_grabbed_list = []
for extension in config.backup_extenstion:
full_glob_path = os.path.join(full_dir_path, extension)
files_grabbed_list.extend(glob.glob(full_glob_path))
count_file = len(files_grabbed_list)
time_period = os.path.split(full_dir_path)[1]
if int(store_backup_count):
delta_count_file = int(count_file) - int(store_backup_count)
if ((time_period == 'weekly' and dow != config.dow_backup) or
(time_period == 'monthly' and dom != config.dom_backup)):
result_delete_count = delta_count_file
else:
result_delete_count = delta_count_file + 1
if safety_backup:
result_delete_count -= 1
if result_delete_count < 1:
return 1
try:
delete_oldest_files(files_grabbed_list, result_delete_count, job_name)
except general_function.MyError as err:
if storage == 'local':
log_and_mail.writelog(
'ERROR',
f"Can't delete {files_type} '{time_period}' files in directory '{full_dir_path}' on '{storage}' "
f"storage: {err}",
config.filelog_fd, job_name)
elif storage == 'smb':
log_and_mail.writelog(
'ERROR',
f"Can't delete {files_type} '{time_period}' files in directory '{full_path_for_log}' in '{share}' "
f"share on '{storage}' storage({host}): {err}",
config.filelog_fd, job_name)
else:
log_and_mail.writelog(
'ERROR',
f"Can't delete {files_type} '{time_period}' files in directory '{full_path_for_log}' on '{storage}' "
f"storage({host}): {err}",
config.filelog_fd, job_name)
else:
if storage == 'local':
log_and_mail.writelog(
'INFO',
f"Successfully deleted {files_type} '{time_period}' files in directory '{full_dir_path}' on '{storage}' "
f"storage.",
config.filelog_fd, job_name)
elif storage == 'smb':
log_and_mail.writelog(
'INFO',
f"Successfully deleted {files_type} '{time_period}' files in directory '{full_path_for_log}' in '{share}' "
f"share on '{storage}' storage({host}).",
config.filelog_fd, job_name)
else:
log_and_mail.writelog(
'INFO',
f"Successfully deleted {files_type} '{time_period}' files in directory '{full_path_for_log}' on '{storage}' "
f"storage({host}).",
config.filelog_fd, job_name)
else:
try:
for i in files_grabbed_list:
general_function.del_file_objects(job_name, i)
except general_function.MyError as err:
if storage == 'local':
log_and_mail.writelog(
'ERROR',
f"Can't delete {files_type} '{time_period}' files in directory '{full_dir_path}' on '{storage}' "
f"storage:{err}",
config.filelog_fd, job_name)
elif storage == 'smb':
log_and_mail.writelog(
'ERROR',
f"Can't delete {files_type} '{time_period}' files in directory '{full_path_for_log}' in '{share}' "
f"share on '{storage}' storage({host}):{err}",
config.filelog_fd, job_name)
else:
log_and_mail.writelog(
'ERROR',
f"Can't delete {files_type} '{time_period}' files in directory '{full_path_for_log}' on '{storage}' "
f"storage({host}):{err}",
config.filelog_fd, job_name)
else:
if storage == 'local':
log_and_mail.writelog(
'INFO',
f"Successfully deleted {files_type} '{time_period}' files in directory '{full_dir_path}' on '{storage}' "
f"storage.",
config.filelog_fd, job_name)
elif storage == 'smb':
log_and_mail.writelog(
'INFO',
f"Successfully deleted {files_type} '{time_period}' files in directory '{full_path_for_log}' in '{share}' "
f"share on '{storage}' storage({host}).",
config.filelog_fd, job_name)
else:
log_and_mail.writelog(
'INFO',
f"Successfully deleted {files_type} '{time_period}' files in directory '{full_path_for_log}' on '{storage}' "
f"storage({host}).",
config.filelog_fd, job_name)
def desc_files_backup(job_data):
""" Function, creates a desc backup of directories.
At the entrance receives a dictionary with the data of the job.
"""
is_prams_read, job_name, backup_type, tmp_dir, sources, storages, safety_backup, deferred_copying_level = \
general_function.get_job_parameters(job_data)
if not is_prams_read:
return
full_path_tmp_dir = general_function.get_tmp_dir(tmp_dir, backup_type)
dumped_ofs = {}
for i in range(len(sources)):
exclude_list = sources[i].get('excludes', '')
try:
target_list = sources[i]['target']
gzip = sources[i]['gzip']
except KeyError as e:
log_and_mail.writelog('ERROR', f"Missing required key:'{e}'!",
config.filelog_fd, job_name)
continue
# Keeping an exception list in the global variable due to the specificity of
# the `filter` key of the `add` method of the `tarfile` class
general_files_func.EXCLUDE_FILES = general_files_func.get_exclude_ofs(
target_list, exclude_list)
# The backup name is selected depending on the particular glob patterns from
# the list `target_list`
for regex in target_list:
target_ofs_list = general_files_func.get_ofs(regex)
if not target_ofs_list:
log_and_mail.writelog(
'ERROR', "No file system objects found that" +
f"match the regular expression '{regex}'!",
config.filelog_fd, job_name)
continue
for ofs in target_ofs_list:
# Create a backup only if the directory is not in the exception list
# so as not to generate empty backups
if not general_files_func.is_excluded_ofs(ofs):
# A function that by regularity returns the name of
# the backup WITHOUT EXTENSION AND DATE
backup_file_name = general_files_func.get_name_files_backup(
regex, ofs)
# Get the part of the backup storage path for this archive relative to
# the backup dir
part_of_dir_path = backup_file_name.replace('___', '/')
backup_full_tmp_path = general_function.get_full_path(
full_path_tmp_dir, backup_file_name, 'tar', gzip)
periodic_backup.remove_old_local_file(
storages, part_of_dir_path, job_name)
if general_files_func.create_tar('files',
backup_full_tmp_path, ofs,
gzip, backup_type,
job_name):
dumped_ofs[ofs] = {
'success': True,
'tmp_path': backup_full_tmp_path,
'part_of_dir_path': part_of_dir_path
}
else:
dumped_ofs[ofs] = {'success': False}
if deferred_copying_level <= 0 and dumped_ofs[ofs][
'success']:
periodic_backup.general_desc_iteration(
backup_full_tmp_path, storages, part_of_dir_path,
job_name, safety_backup)
else:
continue
for ofs, result in dumped_ofs.items():
if deferred_copying_level == 1 and result['success']:
periodic_backup.general_desc_iteration(
result['tmp_path'], storages,
result['part_of_dir_path'], job_name, safety_backup)
for ofs, result in dumped_ofs.items():
if deferred_copying_level == 2 and result['success']:
periodic_backup.general_desc_iteration(
result['tmp_path'], storages, result['part_of_dir_path'],
job_name, safety_backup)
for ofs, result in dumped_ofs.items():
if deferred_copying_level >= 3 and result['success']:
periodic_backup.general_desc_iteration(result['tmp_path'],
storages,
result['part_of_dir_path'],
job_name, safety_backup)
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type, full_path_tmp_dir,
'/var/cache/davfs2/*')
def mysql_backup(job_data):
is_prams_read, job_name, backup_type, tmp_dir, sources, storages, safety_backup, deferred_copying_level = \
general_function.get_job_parameters(job_data)
if not is_prams_read:
return
full_path_tmp_dir = general_function.get_tmp_dir(tmp_dir, backup_type)
dumped_dbs = {}
for i in range(len(sources)):
exclude_list = sources[i].get('excludes', [])
try:
connect = sources[i]['connect']
target_list = sources[i]['target']
gzip = sources[i]['gzip']
is_slave = sources[i]['is_slave']
extra_keys = sources[i]['extra_keys']
except KeyError as e:
log_and_mail.writelog('ERROR', f"Missing required key:'{e}'!",
config.filelog_fd, job_name)
continue
db_host = connect.get('db_host')
db_port = connect.get('db_port')
socket = connect.get('socket')
db_user = connect.get('db_user')
db_password = connect.get('db_password')
auth_file = connect.get('auth_file')
if not (auth_file or ((db_host or socket) or db_user or db_password)):
log_and_mail.writelog(
'ERROR',
"Can't find the authentication data, please fill in the required fields",
config.filelog_fd, job_name)
continue
if not db_port:
db_port = general_function.get_default_port('mysql')
is_all_flag = False
if 'all' in target_list:
is_all_flag = True
connection_1, str_auth = get_connection(db_host, db_port, db_user,
db_password, auth_file, socket,
job_name)
if connection_1 is None:
continue
cur_1 = connection_1.cursor()
if is_all_flag:
cur_1.execute("SHOW DATABASES")
target_list = [i[0] for i in cur_1.fetchall()]
if is_slave:
try:
cur_1.execute("STOP SLAVE")
except MySQLdb.Error as err:
log_and_mail.writelog('ERROR', f"Can't stop slave: {err}",
config.filelog_fd, job_name)
connection_1.close()
for db in target_list:
if db not in exclude_list:
backup_full_tmp_path = general_function.get_full_path(
full_path_tmp_dir, db, 'sql', gzip, i)
periodic_backup.remove_old_local_file(storages, db, job_name)
if is_success_mysqldump(db, extra_keys, str_auth,
backup_full_tmp_path, gzip, job_name):
dumped_dbs[db] = {
'success': True,
'tmp_path': backup_full_tmp_path
}
else:
dumped_dbs[db] = {'success': False}
if deferred_copying_level <= 0 and dumped_dbs[db]['success']:
periodic_backup.general_desc_iteration(
backup_full_tmp_path, storages, db, job_name,
safety_backup)
if is_slave:
connection_2, str_auth = get_connection(db_host, db_port, db_user,
db_password, auth_file,
socket, job_name)
if connection_2 is None:
log_and_mail.writelog(
'ERROR', f"Can't start slave: Can't connect to MySQL.",
config.filelog_fd, job_name)
return
cur_2 = connection_2.cursor()
try:
cur_2.execute("START SLAVE")
except MySQLdb.Error as err:
log_and_mail.writelog('ERROR', f"Can't start slave: {err} ",
config.filelog_fd, job_name)
connection_2.close()
for db, result in dumped_dbs.items():
if deferred_copying_level == 1 and result['success']:
periodic_backup.general_desc_iteration(result['tmp_path'],
storages, db, job_name,
safety_backup)
for db, result in dumped_dbs.items():
if deferred_copying_level >= 2 and result['success']:
periodic_backup.general_desc_iteration(result['tmp_path'],
storages, db, job_name,
safety_backup)
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type, full_path_tmp_dir,
'/var/cache/davfs2/*')
def create_inc_backup(local_dst_dirname, remote_dir, part_of_dir_path, backup_file_name,
target, exclude_list, gzip, job_name, storage, host, share, months_to_store):
""" The function determines whether to collect a full backup or incremental,
prepares all the necessary information.
"""
date_year = general_function.get_time_now('year')
date_month = general_function.get_time_now('moy')
date_day = general_function.get_time_now('dom')
dated_paths = get_dated_paths(local_dst_dirname, part_of_dir_path, date_year, date_month, date_day)
# Before we proceed to collect a copy, we need to delete the copies for the same month last year
# if they are to not save extra archives
old_month_dirs = []
if os.path.isdir(dated_paths['old_year_dir']) or months_to_store < 12:
if months_to_store < 12:
int_date_month = int(date_month)
last_month = int_date_month - months_to_store
if last_month <= 0:
m_range = list(range(last_month+12, 13))
m_range.extend(list(range(1, int_date_month)))
else:
m_range = list(range(last_month, int_date_month))
for i in range(1, 13):
if i not in m_range:
date = str(i).zfill(2)
if i < int(date_month):
year_to_cleanup = dated_paths['year_dir']
else:
year_to_cleanup = dated_paths['old_year_dir']
old_month_dirs.append(os.path.join(year_to_cleanup, f'month_{date}'))
else:
old_month_dirs.append(os.path.join(dated_paths['old_year_dir'], f'month_{date_month}'))
del_old_inc_file(dated_paths['old_year_dir'], old_month_dirs)
link_dict = {} # dict for symlink with pairs like dst: src
copy_dict = {} # dict for copy with pairs like dst: src
# Get the current list of files
new_meta_info = get_index(target, exclude_list)
if not os.path.isfile(dated_paths['year_inc_file']):
# There is no original index file, so we need to check the existence of an year directory
if os.path.isdir(dated_paths['year_dir']):
# There is a directory, but there is no file itself, then something went wrong, so
# we delete this directory with all the data inside, because even if they are there
# continue to collect incremental copies it will not be able to
general_function.del_file_objects(job_name, dated_paths['year_dir'])
dirs_for_log = general_function.get_dirs_for_log(dated_paths['year_dir'], remote_dir, storage)
file_for_log = os.path.join(dirs_for_log, os.path.basename(dated_paths['year_inc_file']))
log_and_mail.writelog('ERROR',
f"The file {file_for_log} not found, so the directory {dirs_for_log} is cleared. "
f"Incremental backup will be reinitialized ",
config.filelog_fd, job_name)
# Initialize the incremental backup, i.e. collect a full copy
remote_dir_for_logs = general_function.get_dirs_for_log(dated_paths['initial_dir'], remote_dir, storage)
general_function.create_dirs(job_name=job_name, dirs_pairs={dated_paths['initial_dir']: remote_dir_for_logs})
write_meta_info(dated_paths['year_inc_file'], new_meta_info)
full_backup_path = general_function.get_full_path(dated_paths['initial_dir'],
backup_file_name,
'tar',
gzip)
general_files_func.create_tar('files', full_backup_path, target,
gzip, 'inc_files', job_name,
remote_dir, storage, host, share)
daily_dirs_remote = general_function.get_dirs_for_log(dated_paths['daily_dir'], remote_dir, storage)
month_dirs_remote = general_function.get_dirs_for_log(dated_paths['month_dir'], remote_dir, storage)
general_function.create_dirs(job_name=job_name, dirs_pairs={dated_paths['daily_dir']: daily_dirs_remote,
dated_paths['month_dir']: month_dirs_remote})
if storage in 'local':
link_dict[dated_paths['month_inc_file']] = dated_paths['year_inc_file']
link_dict[os.path.join(dated_paths['month_dir'], os.path.basename(full_backup_path))] = full_backup_path
link_dict[dated_paths['daily_inc_file']] = dated_paths['year_inc_file']
link_dict[os.path.join(dated_paths['daily_dir'], os.path.basename(full_backup_path))] = full_backup_path
elif storage in 'scp, nfs':
copy_dict[dated_paths['month_inc_file']] = dated_paths['year_inc_file']
link_dict[os.path.join(dated_paths['month_dir'], os.path.basename(full_backup_path))] = \
full_backup_path.replace(local_dst_dirname, remote_dir)
copy_dict[dated_paths['daily_inc_file']] = dated_paths['year_inc_file']
link_dict[os.path.join(dated_paths['daily_dir'], os.path.basename(full_backup_path))] = \
full_backup_path.replace(local_dst_dirname, remote_dir)
else:
copy_dict[dated_paths['month_inc_file']] = dated_paths['year_inc_file']
copy_dict[os.path.join(dated_paths['month_dir'], os.path.basename(full_backup_path))] = full_backup_path
copy_dict[dated_paths['daily_inc_file']] = dated_paths['year_inc_file']
copy_dict[os.path.join(dated_paths['daily_dir'], os.path.basename(full_backup_path))] = full_backup_path
else:
symlink_dir = ''
meta_path = ''
if int(date_day) == 1:
meta_path = dated_paths['month_inc_file']
old_meta_path = dated_paths['year_inc_file']
general_inc_backup_dir = dated_paths['month_dir']
symlink_dir = dated_paths['daily_dir']
elif int(date_day) == 11 or int(date_day) == 21:
meta_path = dated_paths['daily_inc_file']
old_meta_path = dated_paths['month_inc_file']
general_inc_backup_dir = dated_paths['daily_dir']
else:
old_meta_path = dated_paths['daily_inc_file']
general_inc_backup_dir = dated_paths['daily_dir']
try:
old_meta_info = specific_function.parser_json(old_meta_path)
except general_function.MyError as e:
log_and_mail.writelog('ERROR',
f"Couldn't open old meta info file '{old_meta_path}': {e}!",
config.filelog_fd, job_name)
return 2
general_dirs_for_log = general_function.get_dirs_for_log(general_inc_backup_dir, remote_dir, storage)
general_function.create_dirs(job_name=job_name, dirs_pairs={general_inc_backup_dir: general_dirs_for_log})
if meta_path:
write_meta_info(meta_path, new_meta_info)
# Calculate the difference between the old and new file states
diff_json = compute_diff(new_meta_info, old_meta_info)
# Define the list of files that need to be included in the archive
target_change_list = diff_json['modify']
dict_directory = get_dict_directory(target, diff_json)
inc_backup_path = general_function.get_full_path(general_inc_backup_dir, backup_file_name, 'tar', gzip)
create_inc_tar(
inc_backup_path, remote_dir, dict_directory, target_change_list, gzip, job_name, storage, host, share
)
if symlink_dir:
symlink_dirs_for_log = general_function.get_dirs_for_log(symlink_dir, remote_dir, storage)
general_function.create_dirs(job_name=job_name, dirs_pairs={symlink_dir: symlink_dirs_for_log})
if storage in 'local':
link_dict[dated_paths['daily_inc_file']] = dated_paths['month_inc_file']
elif storage in 'scp, nfs':
copy_dict[dated_paths['daily_inc_file'].replace(local_dst_dirname, remote_dir)] = \
dated_paths['month_inc_file'].replace(local_dst_dirname, remote_dir)
else:
copy_dict[dated_paths['daily_inc_file']] = dated_paths['month_inc_file']
create_links_and_copies(link_dict, copy_dict, job_name)
def control_old_files(full_dir_path,
store_backup_count,
storage,
job_name,
host='',
full_path_for_log='',
share=''):
dow = general_function.get_time_now("dow")
dom = general_function.get_time_now("dom")
files_grabbed_list = []
for extension in config.backup_extenstion:
full_glob_path = os.path.join(full_dir_path, extension)
files_grabbed_list.extend(glob.glob(full_glob_path))
count_file = len(files_grabbed_list)
time_period = os.path.split(full_dir_path)[1]
if int(store_backup_count):
delta_count_file = int(count_file) - int(store_backup_count)
if ((time_period == 'weekly' and dow != config.dow_backup)
or (time_period == 'monthly' and dom != config.dom_backup)):
result_delete_count = delta_count_file
else:
result_delete_count = delta_count_file + 1
if result_delete_count < 1:
return 1
try:
delete_oldest_files(files_grabbed_list, result_delete_count,
job_name)
except general_function.MyError as err:
if storage == 'local':
log_and_mail.writelog(
'ERROR',
"Can't delete old '%s' files in directory '%s' on '%s' storage:%s"
% (time_period, full_dir_path, storage, err),
config.filelog_fd, job_name)
elif storage == 'smb':
log_and_mail.writelog(
'ERROR',
"Can't delete old '%s' files in directory '%s' in '%s' share on '%s' storage(%s):%s"
% (time_period, full_path_for_log, share, storage, host,
err), config.filelog_fd, job_name)
else:
log_and_mail.writelog(
'ERROR',
"Can't delete old '%s' files in directory '%s' on '%s' storage(%s):%s"
% (time_period, full_path_for_log, storage, host, err),
config.filelog_fd, job_name)
else:
if storage == 'local':
log_and_mail.writelog(
'INFO',
"Successfully deleted old '%s' files in directory '%s' on '%s' storage."
% (time_period, full_dir_path, storage), config.filelog_fd,
job_name)
elif storage == 'smb':
log_and_mail.writelog(
'INFO',
"Successfully deleted old '%s' files in directory '%s' in '%s' share on '%s' storage(%s)."
% (time_period, full_path_for_log, share, storage, host),
config.filelog_fd, job_name)
else:
log_and_mail.writelog(
'INFO',
"Successfully deleted old '%s' files in directory '%s' on '%s' storage(%s)."
% (time_period, full_path_for_log, storage, host),
config.filelog_fd, job_name)
else:
try:
for i in files_grabbed_list:
general_function.del_file_objects(job_name, i)
except general_function.MyError as err:
if storage == 'local':
log_and_mail.writelog(
'ERROR',
"Can't delete old '%s' files in directory '%s' on '%s' storage:%s"
% (time_period, full_dir_path, storage, err),
config.filelog_fd, job_name)
elif storage == 'smb':
log_and_mail.writelog(
'ERROR',
"Can't delete old '%s' files in directory '%s' in '%s' share on '%s' storage(%s):%s"
% (time_period, full_path_for_log, share, storage, host,
err), config.filelog_fd, job_name)
else:
log_and_mail.writelog(
'ERROR',
"Can't delete old '%s' files in directory '%s' on '%s' storage(%s):%s"
% (time_period, full_path_for_log, storage, host, err),
config.filelog_fd, job_name)
else:
if storage == 'local':
log_and_mail.writelog(
'INFO',
"Successfully deleted old '%s' files in directory '%s' on '%s' storage."
% (time_period, full_dir_path, storage), config.filelog_fd,
job_name)
elif storage == 'smb':
log_and_mail.writelog(
'INFO',
"Successfully deleted old '%s' files in directory '%s' in '%s' share on '%s' storage(%s)."
% (time_period, full_path_for_log, share, storage, host),
config.filelog_fd, job_name)
else:
log_and_mail.writelog(
'INFO',
"Successfully deleted old '%s' files in directory '%s' on '%s' storage(%s)."
% (time_period, full_path_for_log, storage, host),
config.filelog_fd, job_name)
def desc_files_backup(job_data):
''' Function, creates a desc backup of directories.
At the entrance receives a dictionary with the data of the job.
'''
try:
job_name = job_data['job']
backup_type = job_data['type']
tmp_dir = job_data['tmp_dir']
sources = job_data['sources']
storages = job_data['storages']
except KeyError as e:
log_and_mail.writelog('ERROR', f"Missing required key:'{e}'!",
config.filelog_fd, job_name)
return 1
full_path_tmp_dir = general_function.get_tmp_dir(tmp_dir, backup_type)
for i in range(len(sources)):
exclude_list = sources[i].get('excludes', '')
try:
target_list = sources[i]['target']
gzip = sources[i]['gzip']
except KeyError as e:
log_and_mail.writelog('ERROR', f"Missing required key:'{e}'!",
config.filelog_fd, job_name)
continue
# Keeping an exception list in the global variable due to the specificity of
# the `filter` key of the `add` method of the `tarfile` class
general_files_func.EXCLUDE_FILES = general_files_func.get_exclude_ofs(target_list,
exclude_list)
# The backup name is selected depending on the particular glob patterns from
# the list `target_list`
for regex in target_list:
target_ofs_list = general_files_func.get_ofs(regex)
if not target_ofs_list:
log_and_mail.writelog('ERROR', "No file system objects found that" +\
f"match the regular expression '{regex}'!",
config.filelog_fd, job_name)
continue
for i in target_ofs_list:
# Create a backup only if the directory is not in the exception list
# so as not to generate empty backups
if not general_files_func.is_excluded_ofs(i):
# A function that by regularity returns the name of
# the backup WITHOUT EXTENSION AND DATE
backup_file_name = general_files_func.get_name_files_backup(regex, i)
# Get the part of the backup storage path for this archive relative to
# the backup dir
part_of_dir_path = backup_file_name.replace('___', '/')
backup_full_tmp_path = general_function.get_full_path(
full_path_tmp_dir,
backup_file_name,
'tar',
gzip)
periodic_backup.remove_old_local_file(storages, part_of_dir_path, job_name)
if general_files_func.create_tar('files', backup_full_tmp_path, i,
gzip, backup_type, job_name):
# If the dump collection in the temporary directory has successfully
# transferred the data to the specified storage
periodic_backup.general_desc_iteration(backup_full_tmp_path,
storages, part_of_dir_path,
job_name)
else:
continue
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type,
full_path_tmp_dir, '/var/cache/davfs2/*')
def postgresql_basebackup(job_data):
try:
job_name = job_data['job']
backup_type = job_data['type']
tmp_dir = job_data['tmp_dir']
sources = job_data['sources']
storages = job_data['storages']
except KeyError as e:
log_and_mail.writelog('ERROR', "Missing required key:'%s'!" %(e), config.filelog_fd, job_name)
return 1
full_path_tmp_dir = general_function.get_tmp_dir(tmp_dir, backup_type)
for i in range(len(sources)):
try:
connect = sources[i]['connect']
gzip = sources[i]['gzip']
extra_keys = sources[i]['extra_keys']
except KeyError as e:
log_and_mail.writelog('ERROR', "Missing required key:'%s'!" %(e), config.filelog_fd, job_name)
continue
db_host = connect.get('db_host')
db_port = connect.get('db_port')
db_user = connect.get('db_user')
db_password = connect.get('db_password')
if not (db_user and db_host and db_password):
log_and_mail.writelog('ERROR', "Can't find the authentication data, please fill in the required fields",
config.filelog_fd, job_name)
continue
if not db_port:
db_port = general_function.get_default_port('postgresql')
try:
connection = psycopg2.connect(dbname="postgres", user=db_user, password=db_password, host=db_host, port=db_port)
except psycopg2.Error as err:
log_and_mail.writelog('ERROR', "Can't connect to PostgreSQL instances with with following data host='%s', port='%s', user='******', passwd='%s':%s" %(db_host, db_port, db_user, db_password, err),
config.filelog_fd, job_name)
continue
else:
connection.close()
backup_full_tmp_path = general_function.get_full_path(
full_path_tmp_dir,
'postgresq_hot',
'tar',
gzip)
periodic_backup.remove_old_local_file(storages, '', job_name)
str_auth = ' --dbname=postgresql://%s:%s@%s:%s/ ' %(db_user, db_password, db_host, db_port)
if is_success_pgbasebackup(extra_keys, str_auth, backup_full_tmp_path, gzip, job_name):
periodic_backup.general_desc_iteration(backup_full_tmp_path,
storages, '',
job_name)
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type,
full_path_tmp_dir, '/var/cache/davfs2/*')
def redis_backup(job_data):
is_prams_read, job_name, backup_type, tmp_dir, sources, storages, safety_backup, deferred_copying_level = \
general_function.get_job_parameters(job_data)
if not is_prams_read:
return
full_path_tmp_dir = general_function.get_tmp_dir(tmp_dir, backup_type)
for i in range(len(sources)):
try:
connect = sources[i]['connect']
gzip = sources[i]['gzip']
except KeyError as e:
log_and_mail.writelog('ERROR', f"Missing required key:'{e}'!", config.filelog_fd, job_name)
continue
db_host = connect.get('db_host')
db_port = connect.get('db_port')
db_password = connect.get('db_password')
socket = connect.get('socket')
if not (db_host or socket):
log_and_mail.writelog('ERROR', "Can't find the authentication data, please fill in the required fields",
config.filelog_fd, job_name)
continue
if not db_port:
db_port = general_function.get_default_port('redis')
try:
if db_host:
if db_password:
redis.StrictRedis(host=db_host, port=db_port, password=db_password)
str_auth = f" -h {db_host} -p {db_port} -a '{db_password}' "
else:
redis.StrictRedis(host=db_host, port=db_port)
str_auth = f" -h {db_host} -p {db_port} "
else:
if db_password:
redis.StrictRedis(unix_socket_path=socket, password=db_password)
str_auth = f" -s {socket} -a '{db_password}' "
else:
redis.StrictRedis(unix_socket_path=socket)
str_auth = f" -s {socket} "
except (redis.exceptions.ConnectionError, ConnectionRefusedError) as err:
log_and_mail.writelog('ERROR',
f"Can't connect to Redis instances with with following data host='{db_host}', "
f"port='{db_port}', passwd='{db_password}', socket='{socket}': {err}",
config.filelog_fd, job_name)
continue
else:
backup_full_tmp_path = general_function.get_full_path(
full_path_tmp_dir,
'redis',
'rdb',
gzip)
periodic_backup.remove_old_local_file(storages, '', job_name)
if is_success_bgsave(str_auth, backup_full_tmp_path, gzip, job_name):
periodic_backup.general_desc_iteration(backup_full_tmp_path,
storages, '',
job_name, safety_backup)
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type,
full_path_tmp_dir, '/var/cache/davfs2/*')
def postgresql_backup(job_data):
is_prams_read, job_name, options = general_function.get_job_parameters(
job_data)
if not is_prams_read:
return
full_path_tmp_dir = general_function.get_tmp_dir(options['tmp_dir'],
options['backup_type'])
dumped_dbs = {}
for i in range(len(options['sources'])):
exclude_list = options['sources'][i].get('excludes', [])
try:
connect = options['sources'][i]['connect']
target_list = options['sources'][i]['target']
gzip = options['sources'][i]['gzip']
extra_keys = options['sources'][i]['extra_keys']
except KeyError as e:
log_and_mail.writelog('ERROR', f"Missing required key:'{e}'!",
config.filelog_fd, job_name)
continue
db_host = connect.get('db_host')
db_port = connect.get('db_port')
db_user = connect.get('db_user')
db_password = connect.get('db_password')
if not (db_user or db_host or db_password):
log_and_mail.writelog(
'ERROR',
"Can't find the authentication data, please fill in the required fields",
config.filelog_fd, job_name)
continue
if not db_port:
db_port = general_function.get_default_port('postgresql')
is_all_flag = False
if 'all' in target_list:
is_all_flag = True
if is_all_flag:
try:
connection = psycopg2.connect(dbname="postgres",
user=db_user,
password=db_password,
host=db_host,
port=db_port)
except psycopg2.Error as err:
log_and_mail.writelog(
'ERROR',
f"Can't connect to PostgreSQL instances with with following data host='{db_host}', "
f"port='{db_port}', user='******', passwd='{db_password}':{err}",
config.filelog_fd, job_name)
continue
cur = connection.cursor()
cur.execute("select datname from pg_database;")
target_list = [i[0] for i in cur.fetchall()]
connection.close()
for db in target_list:
if db not in exclude_list:
backup_full_tmp_path = general_function.get_full_path(
full_path_tmp_dir, db, 'pgdump.sql', gzip, i)
periodic_backup.remove_local_file(options['storages'], db,
job_name)
str_auth = f' --dbname=postgresql://{db_user}:{db_password}@{db_host}:{db_port}/{db} '
if is_success_pgdump(db, extra_keys, str_auth,
backup_full_tmp_path, gzip, job_name):
dumped_dbs[db] = {
'success': True,
'tmp_path': backup_full_tmp_path
}
else:
dumped_dbs[db] = {'success': False}
if options['deferred_copying_level'] <= 0 and dumped_dbs[db][
'success']:
periodic_backup.general_desc_iteration(
backup_full_tmp_path, options['storages'], db,
job_name, options['safety_backup'])
for db, result in dumped_dbs.items():
if options['deferred_copying_level'] == 1 and result['success']:
periodic_backup.general_desc_iteration(
result['tmp_path'], options['storages'], db, job_name,
options['safety_backup'])
for db, result in dumped_dbs.items():
if options['deferred_copying_level'] >= 2 and result['success']:
periodic_backup.general_desc_iteration(result['tmp_path'],
options['storages'], db,
job_name,
options['safety_backup'])
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(options['backup_type'],
full_path_tmp_dir, '/var/cache/davfs2/*')
def postgresql_backup(job_data):
try:
job_name = job_data['job']
backup_type = job_data['type']
tmp_dir = job_data['tmp_dir']
sources = job_data['sources']
storages = job_data['storages']
except KeyError as e:
log_and_mail.writelog('ERROR', f"Missing required key:'{e}'!",
config.filelog_fd, job_name)
return 1
full_path_tmp_dir = general_function.get_tmp_dir(tmp_dir, backup_type)
for i in range(len(sources)):
exclude_list = sources[i].get('excludes', [])
try:
connect = sources[i]['connect']
target_list = sources[i]['target']
gzip = sources[i]['gzip']
extra_keys = sources[i]['extra_keys']
except KeyError as e:
log_and_mail.writelog('ERROR', f"Missing required key:'{e}'!",
config.filelog_fd, job_name)
continue
db_host = connect.get('db_host')
db_port = connect.get('db_port')
db_user = connect.get('db_user')
db_password = connect.get('db_password')
if not (db_user and db_host and db_password):
log_and_mail.writelog(
'ERROR',
"Can't find the authentication data, please fill in the required fields",
config.filelog_fd, job_name)
continue
if not db_port:
db_port = general_function.get_default_port('postgresql')
is_all_flag = False
if 'all' in target_list:
is_all_flag = True
if is_all_flag:
try:
connection = psycopg2.connect(dbname="postgres",
user=db_user,
password=db_password,
host=db_host,
port=db_port)
except psycopg2.Error as err:
log_and_mail.writelog(
'ERROR',
f"Can't connect to PostgreSQL instances with with following data host='{db_host}', port='{db_port}', user='******', passwd='{db_password}':{err}",
config.filelog_fd, job_name)
continue
cur = connection.cursor()
cur.execute("select datname from pg_database;")
target_list = [i[0] for i in cur.fetchall()]
connection.close()
for db in target_list:
if not db in exclude_list:
backup_full_tmp_path = general_function.get_full_path(
full_path_tmp_dir, db, 'pgdump', gzip)
periodic_backup.remove_old_local_file(storages, db, job_name)
str_auth = f' --dbname=postgresql://{db_user}:{db_password}@{db_host}:{db_port}/{db} '
if is_success_pgdump(db, extra_keys, str_auth,
backup_full_tmp_path, gzip, job_name):
periodic_backup.general_desc_iteration(
backup_full_tmp_path, storages, db, job_name)
# After all the manipulations, delete the created temporary directory and
# data inside the directory with cache davfs, but not the directory itself!
general_function.del_file_objects(backup_type, full_path_tmp_dir,
'/var/cache/davfs2/*')
