def unmount(storage):
if storage != 'local' and mount_point:
umount_cmd = f"fusermount -uz {mount_point}"
umount = general_function.exec_cmd(umount_cmd)
stderr_umount = umount['stderr']
code = umount['code']
if stderr_umount:
raise general_function.MyError(stderr_umount)
elif code != 0:
raise general_function.MyError(
f"Bad result code external process '{umount_cmd}':'{code}'")
else:
general_function.del_file_objects('', mount_point)
return
def unmount():
if mount_point:
umount_cmd = "fusermount -uz %s" % (mount_point)
umount = general_function.exec_cmd(umount_cmd)
stderr_umount = umount['stderr']
code = umount['code']
if stderr_umount:
raise general_function.MyError(stderr_umount)
elif code != 0:
raise general_function.MyError(
"Bad result code external process '%s':'%s'" %
(umount_cmd, code))
else:
general_function.del_file_objects('', mount_point)
return 1
def parser_json(json_file):
try:
parsed_str = json.load(open(json_file))
except (PermissionError, OSError) as e:
raise general_function.MyError(str(e))
else:
return parsed_str
def get_parsed_string(path_to_config):
try:
with open(path_to_config, 'r') as stream:
try:
yaml_str = yaml.load(stream, Loader=Loader)
except yaml.YAMLError as e:
raise general_function.MyError(str(e))
except RuntimeError as e:
if "maximum recursion depth exceeded while calling" in str(e):
error_msg = f" error in include value - '{e}'"
else:
error_msg = str(e)
raise general_function.MyError(error_msg)
except (FileNotFoundError, PermissionError):
general_function.print_info(
f"No such file '{path_to_config}' or permission denied!")
sys.exit(1)
else:
return yaml_str
def gzip_file(origfile, dstfile):
""" The function compresses the file (used in redis backup).
"""
gzip_tarfile = dstfile
try:
with open(origfile, 'rb') as f_in:
with gzip.open(gzip_tarfile, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
except Exception as e:
raise general_function.MyError(str(e))
def mount(current_storage_data):
""" A function that is responsible for directly mounting a particular storage.
The input receives a dictionary containing the necessary data for connecting storage.
"""
try:
(data_mount, pre_mount) = get_mount_data(current_storage_data)
except MountError as e:
raise general_function.MyError(f"{e}")
if not data_mount:
# if local storage
return
else:
type_storage = data_mount.get('type_storage')
packets = data_mount.get('packets')
check_cmd = data_mount.get('check_cmd')
mount_cmd = data_mount.get('mount_cmd')
for i in packets:
if i:
check_packet = general_function.exec_cmd(f"{check_cmd} {i}")
stdout_check = check_packet['stdout']
if not stdout_check:
raise general_function.MyError(
f"Required package '{i}' not installed!")
else:
continue
if pre_mount:
for key in pre_mount:
try:
f = globals()[key]
args = pre_mount[key]
f(args)
except Exception as err:
raise general_function.MyError(
f"Impossible perform pre-mount operations for storage '{type_storage}': {err}"
)
check_mount_cmd = f"mount | grep {mount_point}"
check_mount = general_function.exec_cmd(check_mount_cmd)
stdout_mount = check_mount['stdout']
if stdout_mount:
if mount_point == '/mnt/sshfs':
remote_mount = stdout_mount.split()[0]
if remote_mount not in mount_cmd:
raise general_function.MyError(
f"Mount point {mount_point} is busy by different remote resource! "
f"Requested mount: {mount_cmd}. "
f"Current mount: {stdout_mount}.")
else:
raise general_function.MyError(
f"Mount point {mount_point} is busy!")
else:
general_function.create_dirs(job_name='',
dirs_pairs={mount_point: ''})
data_mounting = general_function.exec_cmd(f"{mount_cmd}")
stderr_mounting = data_mounting['stderr']
code = data_mounting['code']
if stderr_mounting:
raise general_function.MyError(stderr_mounting)
if code != 0:
raise general_function.MyError(
f"Bad result code external process '{mount_cmd}':'{code}'")
if type_storage == 's3':
try:
os.chdir('/mnt/s3')
except ConnectionAbortedError:
raise general_function.MyError(
"incorrect authentification data!")
else:
os.chdir(
'/'
) # fix error 'getcwd: cannot access parent directories: No such file or directory'
return
def get_storage_data(job_name, storage_data):
""" The function on the input gets the name of the job, as well as the dictionary with this particular storage.
Then it checks all the necessary data for mounting this type of storage to the server. Returns:
filtered dictionary - if successful
Exception MyError with error text in case of problems.
"""
data_dict = {}
storage = storage_data['storage']
data_dict['storage'] = storage
backup_dir = storage_data['backup_dir']
data_dict['backup_dir'] = backup_dir
err_message = ''
if storage not in ('local', 's3'):
host = storage_data.get('host')
if not host:
err_message = f"Field 'host' in job '{job_name}' for storage '{job_name}' can't be empty!"
else:
data_dict['host'] = host
if storage not in ('local', 'nfs', 's3'):
user = storage_data.get('user', '')
password = storage_data.get('password', '')
port = storage_data.get('port', '')
if port:
data_dict['port'] = port
if not user:
err_message = f"Field 'user' in job '{job_name}' for storage '{storage}' can't be empty!"
else:
data_dict['user'] = user
if storage == 'scp':
data_dict['remote_mount_point'] = storage_data.get(
'remote_mount_point', backup_dir)
path_to_key = storage_data.get('path_to_key', '')
if not (password or path_to_key):
err_message = "At least one of the fields 'path_to_key' or 'password' must be filled in" + \
f" job '{job_name}' for storage '{storage}'!"
else:
if password:
data_dict['password'] = password
if path_to_key:
data_dict['path_to_key'] = path_to_key
else:
if not password:
err_message = f"Field 'password' in job '{job_name}' for storage '{storage}' can't be empty!"
else:
data_dict['password'] = password
if storage == 'nfs':
data_dict['remote_mount_point'] = storage_data.get(
'remote_mount_point', backup_dir)
data_dict['extra_keys'] = storage_data.get('extra_keys', '')
if storage == 'smb':
share = storage_data.get('share', '')
if not share:
err_message = f"Field 'share' in job '{job_name}' for storage '{storage}' can't be empty!"
else:
data_dict['share'] = share
if storage == 's3':
bucketname = storage_data.get('bucket_name', '')
if not bucketname:
err_message = f"Field 'bucketname' in job '{job_name}' for storage '{storage}' can't be empty!"
else:
data_dict['bucket_name'] = bucketname
data_dict['s3fs_opts'] = storage_data.get('s3fs_opts', '')
data_dict['access_key_id'] = storage_data.get('access_key_id', '')
data_dict['secret_access_key'] = storage_data.get(
'secret_access_key', '')
if err_message:
raise general_function.MyError(err_message)
else:
return data_dict
def mount(current_storage_data):
''' A function that is responsible for directly mounting a particular storage.
The input receives a dictionary containing the necessary data for connecting storage.
'''
try:
(data_mount, pre_mount) = get_mount_data(current_storage_data)
except MountError as e:
raise general_function.MyError("%s" % e)
if not data_mount:
# if local storage
return 0
else:
type_storage = data_mount.get('type_storage')
packets = data_mount.get('packets')
check_cmd = data_mount.get('check_cmd')
mount_cmd = data_mount.get('mount_cmd')
for i in packets:
if i:
check_packet = general_function.exec_cmd("%s %s" %
(check_cmd, i))
stdout_check = check_packet['stdout']
if not stdout_check:
raise general_function.MyError(
"Required package '%s' not installed!" % (i))
else:
continue
if pre_mount:
for key in pre_mount:
try:
f = globals()[key]
args = pre_mount[key]
f(args)
except Exception as err:
raise general_function.MyError(
"Impossible perform pre-mount operations for storage '%s': %s"
% (type_storage, err))
check_mount_cmd = "mount | grep %s" % (mount_point)
check_mount = general_function.exec_cmd(check_mount_cmd)
stdout_mount = check_mount['stdout']
if stdout_mount:
raise general_function.MyError("Mount point %s is busy!" %
(mount_point))
else:
general_function.create_dirs(job_name='',
dirs_pairs={mount_point: ''})
data_mounting = general_function.exec_cmd("%s" % (mount_cmd))
stderr_mounting = data_mounting['stderr']
code = data_mounting['code']
if stderr_mounting:
raise general_function.MyError(stderr_mounting)
if code != 0:
raise general_function.MyError(
"Bad result code external process '%s':'%s'" %
(mount_cmd, code))
if type_storage == 's3':
try:
os.chdir('/mnt/s3')
except ConnectionAbortedError:
raise general_function.MyError(
"incorrect authentification data!")
return 1
def mount(current_storage_data):
''' A function that is responsible for directly mounting a particular storage.
The input receives a dictionary containing the necessary data for connecting storage.
'''
try:
(data_mount, pre_mount) = get_mount_data(current_storage_data)
except MountError as e:
raise general_function.MyError("%s" % e)
if not data_mount:
# if local storage
return 0
else:
packets = data_mount.get('packets')
update_cmd = data_mount.get('update_cmd')
check_cmd = data_mount.get('check_cmd')
install_cmd = data_mount.get('install_cmd')
mount_cmd = data_mount.get('mount_cmd')
pre_install_cmd = data_mount.get('pre_install_cmd')
if packets:
command = general_function.exec_cmd(update_cmd)
code = command['code']
if code != 0:
raise general_function.MyError(
"Bad result code external process '%s':'%s'" %
(update_cmd, code))
for i in packets:
check_packet = general_function.exec_cmd("%s %s" % (check_cmd, i))
stdout_check = check_packet['stdout']
if not stdout_check:
if pre_install_cmd:
pre_install = general_function.exec_cmd(pre_install_cmd)
stderr_pre_install = pre_install['stderr']
code = pre_install['code']
if stderr_pre_install:
raise general_function.MyError(
"Package '%s' can't installed:%s" %
(i, stderr_pre_install))
if code != 0:
raise general_function.MyError(
"Bad result code external process '%s':'%s'" %
(pre_install_cmd, code))
install_packet = general_function.exec_cmd("%s %s" %
(install_cmd, i))
stderr_install = install_packet['stderr']
code = install_packet['code']
if stderr_install:
raise general_function.MyError(
"Package '%s' can't installed:%s" %
(i, stderr_install))
if code != 0:
raise general_function.MyError(
"Bad result code external process '%s':'%s'" %
(install_cmd, code))
if pre_mount:
for key in pre_mount:
try:
f = globals()[key]
args = pre_mount[key]
f(args)
except Exception as err:
raise general_function.MyError(
"Impossible perform pre-mount operations for storage '%s': %s"
% (current_storage_data.get('storage'), err))
check_mount_cmd = "mount | grep %s" % (mount_point)
check_mount = general_function.exec_cmd(check_mount_cmd)
stdout_mount = check_mount['stdout']
if stdout_mount:
raise general_function.MyError("Mount point %s is busy!" %
(mount_point))
else:
general_function.create_dirs(job_name='',
dirs_pairs={mount_point: ''})
data_mounting = general_function.exec_cmd("%s" % (mount_cmd))
stderr_mounting = data_mounting['stderr']
code = data_mounting['code']
if stderr_mounting:
raise general_function.MyError(stderr_mounting)
if code != 0:
raise general_function.MyError(
"Bad result code external process '%s':'%s'" %
(mount_cmd, code))
return 1
