def main():
possible_actions = ['start', 'stop', 'status', 'reload']
arguments.parse_args(possible_actions)
arguments.setup_logging()
if CONF.action is None or CONF.action not in possible_actions:
CONF.print_help()
return 65 # os.EX_DATAERR
apiclient = None
if CONF.no_api is False:
try:
apiclient = client.Client(opts=CONF)
if CONF.client_id:
apiclient.client_id = CONF.client_id
except Exception as e:
LOG.error(e)
print(e)
sys.exit(1)
else:
if winutils.is_windows():
print("--no-api mode is not available on windows")
return 69 # os.EX_UNAVAILABLE
freezer_utils.create_dir(CONF.jobs_dir, do_log=False)
freezer_scheduler = FreezerScheduler(apiclient=apiclient,
interval=int(CONF.interval),
job_path=CONF.jobs_dir,
concurrent_jobs=CONF.concurrent_jobs)
if CONF.no_daemon:
print('Freezer Scheduler running in no-daemon mode')
LOG.debug('Freezer Scheduler running in no-daemon mode')
if winutils.is_windows():
daemon = win_daemon.NoDaemon(daemonizable=freezer_scheduler)
else:
daemon = linux_daemon.NoDaemon(daemonizable=freezer_scheduler)
else:
if winutils.is_windows():
daemon = win_daemon.Daemon(daemonizable=freezer_scheduler,
interval=int(CONF.interval),
job_path=CONF.jobs_dir,
insecure=CONF.insecure,
concurrent_jobs=CONF.concurrent_jobs)
else:
daemon = linux_daemon.Daemon(daemonizable=freezer_scheduler)
if CONF.action == 'start':
daemon.start()
elif CONF.action == 'stop':
daemon.stop()
elif CONF.action == 'reload':
daemon.reload()
elif CONF.action == 'status':
daemon.status()
# os.RETURN_CODES are only available to posix like systems, on windows
# we need to translate the code to an actual number which is the equivalent
return 0 # os.EX_OK
def test_create_dir(self):
dir1 = '/tmp'
dir2 = '/tmp/testnoexistent1234'
dir3 = '~'
assert utils.create_dir(dir1) is None
assert utils.create_dir(dir2) is None
os.rmdir(dir2)
assert utils.create_dir(dir3) is None
def test_create_dir(self):
dir1 = '/tmp'
dir2 = '/tmp/testnoexistent1234'
dir3 = '~'
assert utils.create_dir(dir1) is None
assert utils.create_dir(dir2) is None
os.rmdir(dir2)
assert utils.create_dir(dir3) is None
def do_job_download(client, args):
freezer_utils.create_dir(args.jobs_dir, do_log=True)
for doc in _job_list(client, args):
fname = os.path.normpath('{0}/job_{1}.conf'.
format(args.jobs_dir, doc['job_id']))
try:
utils.save_doc_to_json_file(doc, fname)
except Exception:
print("Unable to write to file {0}".format(fname))
def prepare_logging(log_file='~/.freezer/freezer.log'):
"""
Creates log directory and log file if no log files provided
:return:
"""
expanded_file_name = os.path.expanduser(log_file)
expanded_dir_name = os.path.dirname(expanded_file_name)
utils.create_dir(expanded_dir_name, do_log=False)
return expanded_file_name
def do_job_download(client, args):
freezer_utils.create_dir(args.jobs_dir, do_log=True)
for doc in _job_list(client, args):
fname = os.path.normpath('{0}/job_{1}.conf'.format(
args.jobs_dir, doc['job_id']))
try:
utils.save_doc_to_json_file(doc, fname)
except Exception:
print("Unable to write to file {0}".format(fname))
def prepare_logging(log_file='~/.freezer/freezer.log'):
"""
Creates log directory and log file if no log files provided
:return:
"""
expanded_file_name = os.path.expanduser(log_file)
expanded_dir_name = os.path.dirname(expanded_file_name)
utils.create_dir(expanded_dir_name, do_log=False)
return expanded_file_name
def prepare_logging(log_file=None):
"""
Creates log directory and log file if no log files provided
:return:
"""
if not log_file:
log_file = os.path.join(home, '.freezer', 'freezer.log')
expanded_file_name = os.path.expanduser(log_file)
expanded_dir_name = os.path.dirname(expanded_file_name)
utils.create_dir(expanded_dir_name, do_log=False)
return expanded_file_name
def prepare_logging(log_file=None):
"""
Creates log directory and log file if no log files provided
:return:
"""
if not log_file:
log_file = os.path.join(home, '.freezer', 'freezer.log')
expanded_file_name = os.path.expanduser(log_file)
expanded_dir_name = os.path.dirname(expanded_file_name)
utils.create_dir(expanded_dir_name, do_log=False)
return expanded_file_name
def start(self, log_file=None):
"""Initialize freezer-scheduler instance inside a windows service
"""
setup_logging(log_file)
utils.create_dir(self.home)
if self.insecure:
os.environ['SERVICE_INSECURE'] = 'True'
# send arguments info to the windows service
os.environ['SERVICE_JOB_PATH'] = self.job_path
os.environ['SERVICE_INTERVAL'] = str(self.interval)
winutils.save_environment(self.home)
print('Freezer Service is starting')
win32serviceutil.StartService(self.service_name)
def start(self, log_file=None):
"""Initialize freezer-scheduler instance inside a windows service
"""
setup_logging(log_file)
utils.create_dir(self.home)
if self.insecure:
os.environ['SERVICE_INSECURE'] = 'True'
# send arguments info to the windows service
os.environ['SERVICE_JOB_PATH'] = self.job_path
os.environ['SERVICE_INTERVAL'] = str(self.interval)
winutils.save_environment(self.home)
print('Freezer Service is starting')
win32serviceutil.StartService(self.service_name)
def configure_logging(file_name):
expanded_file_name = os.path.expanduser(file_name)
expanded_dir_name = os.path.dirname(expanded_file_name)
utils.create_dir(expanded_dir_name, do_log=False)
logging.basicConfig(
filename=expanded_file_name,
level=logging.INFO,
format=('%(asctime)s %(name)s %(levelname)s %(message)s'))
# filter out some annoying messages
# not the best position for this code
log_filter = NoLogFilter()
logging.getLogger("apscheduler.scheduler").\
addFilter(log_filter)
logging.getLogger("apscheduler.executors.default").\
addFilter(log_filter)
logging.getLogger("requests.packages.urllib3.connectionpool").\
addFilter(log_filter)
return expanded_file_name
def configure_logging(file_name):
expanded_file_name = os.path.expanduser(file_name)
expanded_dir_name = os.path.dirname(expanded_file_name)
utils.create_dir(expanded_dir_name, do_log=False)
logging.basicConfig(
filename=expanded_file_name,
level=logging.INFO,
format=('%(asctime)s %(name)s %(levelname)s %(message)s'))
# filter out some annoying messages
# not the best position for this code
log_filter = NoLogFilter()
logging.getLogger("apscheduler.scheduler").\
addFilter(log_filter)
logging.getLogger("apscheduler.executors.default").\
addFilter(log_filter)
logging.getLogger("requests.packages.urllib3.connectionpool").\
addFilter(log_filter)
return expanded_file_name
def download_meta_file(self, backup):
"""
Downloads meta_data to work_dir of previous backup.
:type backup: freezer.storage.base.Backup
:param backup: A backup or increment. Current backup is incremental,
that means we should download tar_meta for detection new files and
changes. If backup.tar_meta is false, raise Exception
:return:
"""
utils.create_dir(self.work_dir)
if backup.level == 0:
return utils.path_join(self.work_dir, backup.tar())
meta_backup = backup.full_backup.increments[backup.level - 1]
if not meta_backup.tar_meta:
raise ValueError('Latest update have no tar_meta')
to_path = utils.path_join(self.work_dir, meta_backup.tar())
if os.path.exists(to_path):
os.remove(to_path)
meta_backup.storage.get_file(
meta_backup.storage.meta_file_abs_path(meta_backup), to_path)
return to_path
def download_meta_file(self, backup):
"""
Downloads meta_data to work_dir of previous backup.
:type backup: freezer.storage.base.Backup
:param backup: A backup or increment. Current backup is incremental,
that means we should download tar_meta for detection new files and
changes. If backup.tar_meta is false, raise Exception
:return:
"""
utils.create_dir(self.work_dir)
if backup.level == 0:
return utils.path_join(self.work_dir, backup.tar())
meta_backup = backup.full_backup.increments[backup.level - 1]
if not meta_backup.tar_meta:
raise ValueError('Latest update have no tar_meta')
to_path = utils.path_join(self.work_dir, meta_backup.tar())
if os.path.exists(to_path):
os.remove(to_path)
meta_backup.storage.get_file(
meta_backup.storage.meta_file_abs_path(meta_backup), to_path)
return to_path
def create_dirs(self):
tmpdir = tempfile.mkdtemp()
if self.temp:
backup_dir = tempfile.mkdtemp(dir=tmpdir,
prefix=self.BACKUP_DIR_PREFIX)
files_dir = tempfile.mkdtemp(dir=tmpdir,
prefix=self.FILES_DIR_PREFIX)
work_dir = tempfile.mkdtemp(dir=tmpdir,
prefix=self.WORK_DIR_PREFIX)
else:
backup_dir = tmpdir + self.BACKUP_DIR_PREFIX
files_dir = tmpdir + self.FILES_DIR_PREFIX
work_dir = tmpdir + self.WORK_DIR_PREFIX
utils.create_dir(backup_dir)
utils.create_dir(work_dir)
utils.create_dir(files_dir)
self.create_content(files_dir)
return backup_dir, files_dir, work_dir
def create_dirs(self):
tmpdir = tempfile.mkdtemp()
if self.temp:
backup_dir = tempfile.mkdtemp(
dir=tmpdir, prefix=self.BACKUP_DIR_PREFIX)
files_dir = tempfile.mkdtemp(
dir=tmpdir, prefix=self.FILES_DIR_PREFIX)
work_dir = tempfile.mkdtemp(
dir=tmpdir, prefix=self.WORK_DIR_PREFIX)
else:
backup_dir = tmpdir + self.BACKUP_DIR_PREFIX
files_dir = tmpdir + self.FILES_DIR_PREFIX
work_dir = tmpdir + self.WORK_DIR_PREFIX
utils.create_dir(backup_dir)
utils.create_dir(work_dir)
utils.create_dir(files_dir)
self.create_content(files_dir)
return backup_dir, files_dir, work_dir
def lvm_snap(backup_opt_dict):
"""
Checks the provided parameters and create the lvm snapshot if requested
The path_to_backup might be adjusted in case the user requested
a lvm snapshot without specifying an exact path for the snapshot).
The assumption in this case is that the user wants to use the lvm snapshot
capability to backup the specified filesystem path, leaving out all
the rest of the parameters which will guessed and set by freezer.
:param backup_opt_dict: the configuration dict
:return: True if the snapshot has been taken, False otherwise
"""
lvm_uuid = uuidutils.generate_uuid(dashed=False)
if not backup_opt_dict.lvm_snapname:
backup_opt_dict.lvm_snapname = \
"{0}_{1}".format(freezer_config.DEFAULT_LVM_SNAP_BASENAME,
lvm_uuid)
# adjust/check lvm parameters according to provided path_to_backup
lvm_info = get_lvm_info(backup_opt_dict.path_to_backup)
if not backup_opt_dict.lvm_volgroup:
backup_opt_dict.lvm_volgroup = lvm_info['volgroup']
if not backup_opt_dict.lvm_srcvol:
backup_opt_dict.lvm_srcvol = lvm_info['srcvol']
if not backup_opt_dict.lvm_dirmount:
utils.create_dir(freezer_config.DEFAULT_LVM_MOUNT_BASEDIR)
backup_opt_dict.lvm_dirmount = \
"{0}/mount_{1}".format(freezer_config.DEFAULT_LVM_MOUNT_BASEDIR,
lvm_uuid)
backup_opt_dict.path_to_backup = os.path.join(backup_opt_dict.lvm_dirmount,
lvm_info['snap_path'])
if not validate_lvm_params(backup_opt_dict):
LOG.info('No LVM requested/configured')
return False
utils.create_dir(backup_opt_dict.lvm_dirmount)
if '%' in backup_opt_dict.lvm_snapsize:
lvm_size_option = "--extents"
else:
lvm_size_option = "--size"
lvm_create_command = (
'{0} {1} {2} --snapshot --permission {3} '
'--name {4} {5}'.format(
utils.find_executable('lvcreate'),
lvm_size_option,
backup_opt_dict.lvm_snapsize,
('r' if backup_opt_dict.lvm_snapperm == 'ro'
else backup_opt_dict.lvm_snapperm),
backup_opt_dict.lvm_snapname,
backup_opt_dict.lvm_srcvol))
lvm_process = subprocess.Popen(
lvm_create_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True,
executable=utils.find_executable('bash'))
(lvm_out, lvm_err) = lvm_process.communicate()
if lvm_process.returncode:
raise Exception('lvm snapshot creation error: {0}'.format(lvm_err))
LOG.debug('{0}'.format(lvm_out))
LOG.warning('Logical volume "{0}" created'.
format(backup_opt_dict.lvm_snapname))
# Guess the file system of the provided source volume and st mount
# options accordingly
filesys_type = get_vol_fs_type(backup_opt_dict.lvm_srcvol)
mount_options = '-o {}'.format(backup_opt_dict.lvm_snapperm)
if 'xfs' == filesys_type:
mount_options = ' -onouuid '
# Mount the newly created snapshot to dir_mount
abs_snap_name = '/dev/{0}/{1}'.format(
backup_opt_dict.lvm_volgroup,
backup_opt_dict.lvm_snapname)
mount_command = '{0} {1} {2} {3}'.format(
utils.find_executable('mount'),
mount_options,
abs_snap_name,
backup_opt_dict.lvm_dirmount)
mount_process = subprocess.Popen(
mount_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True,
executable=utils.find_executable('bash'))
mount_err = mount_process.communicate()[1]
if 'already mounted' in mount_err:
LOG.warning('Volume {0} already mounted on {1}\
'.format(abs_snap_name, backup_opt_dict.lvm_dirmount))
return True
if mount_err:
LOG.error("Snapshot mount error. Removing snapshot")
lvm_snap_remove(backup_opt_dict)
raise Exception('lvm snapshot mounting error: {0}'.format(mount_err))
else:
LOG.warning(
'Volume {0} successfully mounted on {1}'.format(
abs_snap_name, backup_opt_dict.lvm_dirmount))
return True
def lvm_snap(backup_opt_dict):
"""
Checks the provided parameters and create the lvm snapshot if requested
The path_to_backup might be adjusted in case the user requested
a lvm snapshot without specifying an exact path for the snapshot).
The assumption in this case is that the user wants to use the lvm snapshot
capability to backup the specified filesystem path, leaving out all
the rest of the parameters which will guessed and set by freezer.
:param backup_opt_dict: the configuration dict
:return: True if the snapshot has been taken, False otherwise
"""
lvm_uuid = uuidutils.generate_uuid(dashed=False)
if not backup_opt_dict.lvm_snapname:
backup_opt_dict.lvm_snapname = \
"{0}_{1}".format(freezer_config.DEFAULT_LVM_SNAP_BASENAME,
lvm_uuid)
# adjust/check lvm parameters according to provided path_to_backup
lvm_info = get_lvm_info(backup_opt_dict.path_to_backup)
if not backup_opt_dict.lvm_volgroup:
backup_opt_dict.lvm_volgroup = lvm_info['volgroup']
if not backup_opt_dict.lvm_srcvol:
backup_opt_dict.lvm_srcvol = lvm_info['srcvol']
if not backup_opt_dict.lvm_dirmount:
utils.create_dir(freezer_config.DEFAULT_LVM_MOUNT_BASEDIR)
backup_opt_dict.lvm_dirmount = \
"{0}/mount_{1}".format(freezer_config.DEFAULT_LVM_MOUNT_BASEDIR,
lvm_uuid)
if not validate_lvm_params(backup_opt_dict):
LOG.info('No LVM requested/configured')
return False
utils.create_dir(backup_opt_dict.lvm_dirmount)
if '%' in backup_opt_dict.lvm_snapsize:
lvm_size_option = "--extents"
else:
lvm_size_option = "--size"
lvm_create_command = (
'{0} {1} {2} --snapshot --permission {3} '
'--name {4} {5}'.format(
utils.find_executable('lvcreate'),
lvm_size_option,
backup_opt_dict.lvm_snapsize,
('r' if backup_opt_dict.lvm_snapperm == 'ro'
else backup_opt_dict.lvm_snapperm),
backup_opt_dict.lvm_snapname,
backup_opt_dict.lvm_srcvol))
lvm_process = subprocess.Popen(
lvm_create_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True,
executable=utils.find_executable('bash'))
(lvm_out, lvm_err) = lvm_process.communicate()
if lvm_process.returncode:
raise Exception('lvm snapshot creation error: {0}'.format(lvm_err))
LOG.debug('{0}'.format(lvm_out))
LOG.warning('Logical volume "{0}" created'.
format(backup_opt_dict.lvm_snapname))
# Guess the file system of the provided source volume and st mount
# options accordingly
filesys_type = get_vol_fs_type(backup_opt_dict.lvm_srcvol)
mount_options = '-o {}'.format(backup_opt_dict.lvm_snapperm)
if 'xfs' == filesys_type:
mount_options = ' -onouuid '
# Mount the newly created snapshot to dir_mount
abs_snap_name = '/dev/{0}/{1}'.format(
backup_opt_dict.lvm_volgroup,
backup_opt_dict.lvm_snapname)
mount_command = '{0} {1} {2} {3}'.format(
utils.find_executable('mount'),
mount_options,
abs_snap_name,
backup_opt_dict.lvm_dirmount)
mount_process = subprocess.Popen(
mount_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True,
executable=utils.find_executable('bash'))
mount_err = mount_process.communicate()[1]
if 'already mounted' in mount_err:
LOG.warning('Volume {0} already mounted on {1}\
'.format(abs_snap_name, backup_opt_dict.lvm_dirmount))
return True
if mount_err:
LOG.error("Snapshot mount error. Removing snapshot")
lvm_snap_remove(backup_opt_dict)
raise Exception('lvm snapshot mounting error: {0}'.format(mount_err))
else:
LOG.warning(
'Volume {0} successfully mounted on {1}'.format(
abs_snap_name, backup_opt_dict.lvm_dirmount))
# After snapshot is mounted, adjust path_to_backup according the mount
# point and relative path of the snapshot volume.
lvm_info = get_lvm_info(backup_opt_dict.path_to_backup)
backup_opt_dict.path_to_backup = os.path.join(backup_opt_dict.lvm_dirmount,
lvm_info['snap_path'])
return True
def lvm_snap(backup_opt_dict):
"""
Checks the provided parameters and create the lvm snapshot if requested
The path_to_backup might be adjusted in case the user requested
a lvm snapshot without specifying an exact path for the snapshot
(lvm_auto_snap).
The assumption in this case is that the user wants to use the lvm snapshot
capability to backup the specified filesystem path, leaving out all
the rest of the parameters which will guessed and set by freezer.
if a snapshot is requested using the --snapshot flag, but lvm_auto_snap
is not provided, then path_to_backup is supposed to be the path to backup
*before* any information about the snapshot is added and will be
adjusted.
:param backup_opt_dict: the configuration dict
:return: True if the snapshot has been taken, False otherwise
"""
if backup_opt_dict.snapshot:
if not backup_opt_dict.lvm_auto_snap:
# 1) the provided path_to_backup has the meaning of
# the lvm_auto_snap and is therefore copied into it
# 2) the correct value of path_to_backup, which takes into
# consideration the snapshot mount-point, is cleared
# and will be calculated by freezer
backup_opt_dict.lvm_auto_snap =\
backup_opt_dict.path_to_backup
backup_opt_dict.path_to_backup = ''
if not backup_opt_dict.lvm_snapname:
backup_opt_dict.lvm_snapname = \
"{0}_{1}".format(freezer_config.DEFAULT_LVM_SNAP_BASENAME,
uuid.uuid4().hex)
if backup_opt_dict.lvm_auto_snap:
# adjust/check lvm parameters according to provided lvm_auto_snap
lvm_info = get_lvm_info(backup_opt_dict.lvm_auto_snap)
if not backup_opt_dict.lvm_volgroup:
backup_opt_dict.lvm_volgroup = lvm_info['volgroup']
if not backup_opt_dict.lvm_srcvol:
backup_opt_dict.lvm_srcvol = lvm_info['srcvol']
if not backup_opt_dict.lvm_dirmount:
backup_opt_dict.lvm_dirmount = \
"{0}_{1}".format(freezer_config.DEFAULT_LVM_MOUNT_BASENAME,
uuid.uuid4().hex)
path_to_backup = os.path.join(backup_opt_dict.lvm_dirmount,
lvm_info['snap_path'])
if backup_opt_dict.path_to_backup:
# path_to_backup is user-provided, check if consistent
if backup_opt_dict.path_to_backup != path_to_backup:
raise Exception('Path to backup mismatch. '
'provided: {0}, should be LVM-mounted: {1}'.
format(backup_opt_dict.path_to_backup,
path_to_backup))
else:
# path_to_backup not provided: use the one calculated above
backup_opt_dict.path_to_backup = path_to_backup
if not validate_lvm_params(backup_opt_dict):
logging.info('[*] No LVM requested/configured')
return False
utils.create_dir(backup_opt_dict.lvm_dirmount)
lvm_create_command = (
'{0} --size {1} --snapshot --permission {2} '
'--name {3} {4}'.format(
utils.find_executable('lvcreate'),
backup_opt_dict.lvm_snapsize,
('r' if backup_opt_dict.lvm_snapperm == 'ro'
else backup_opt_dict.lvm_snapperm),
backup_opt_dict.lvm_snapname,
backup_opt_dict.lvm_srcvol))
lvm_process = subprocess.Popen(
lvm_create_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True,
executable=utils.find_executable('bash'))
(lvm_out, lvm_err) = lvm_process.communicate()
if lvm_process.returncode:
raise Exception('lvm snapshot creation error: {0}'.format(lvm_err))
logging.debug('[*] {0}'.format(lvm_out))
logging.warning('[*] Logical volume "{0}" created'.
format(backup_opt_dict.lvm_snapname))
# Guess the file system of the provided source volume and st mount
# options accordingly
filesys_type = get_vol_fs_type(backup_opt_dict.lvm_srcvol)
mount_options = '-o {}'.format(backup_opt_dict.lvm_snapperm)
if 'xfs' == filesys_type:
mount_options = ' -onouuid '
# Mount the newly created snapshot to dir_mount
abs_snap_name = '/dev/{0}/{1}'.format(
backup_opt_dict.lvm_volgroup,
backup_opt_dict.lvm_snapname)
mount_command = '{0} {1} {2} {3}'.format(
utils.find_executable('mount'),
mount_options,
abs_snap_name,
backup_opt_dict.lvm_dirmount)
mount_process = subprocess.Popen(
mount_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True,
executable=utils.find_executable('bash'))
mount_err = mount_process.communicate()[1]
if 'already mounted' in mount_err:
logging.warning('[*] Volume {0} already mounted on {1}\
'.format(abs_snap_name, backup_opt_dict.lvm_dirmount))
return True
if mount_err:
logging.error("[*] Snapshot mount error. Removing snapshot")
lvm_snap_remove(backup_opt_dict)
raise Exception('lvm snapshot mounting error: {0}'.format(mount_err))
else:
logging.warning(
'[*] Volume {0} succesfully mounted on {1}'.format(
abs_snap_name, backup_opt_dict.lvm_dirmount))
return True
