def main(): possible_actions = ['start', 'stop', 'status', 'reload'] arguments.parse_args(possible_actions) arguments.setup_logging() if CONF.action is None or CONF.action not in possible_actions: CONF.print_help() return 65 # os.EX_DATAERR apiclient = None if CONF.no_api is False: try: apiclient = client.Client(opts=CONF) if CONF.client_id: apiclient.client_id = CONF.client_id except Exception as e: LOG.error(e) print(e) sys.exit(1) else: if winutils.is_windows(): print("--no-api mode is not available on windows") return 69 # os.EX_UNAVAILABLE freezer_utils.create_dir(CONF.jobs_dir, do_log=False) freezer_scheduler = FreezerScheduler(apiclient=apiclient, interval=int(CONF.interval), job_path=CONF.jobs_dir, concurrent_jobs=CONF.concurrent_jobs) if CONF.no_daemon: print('Freezer Scheduler running in no-daemon mode') LOG.debug('Freezer Scheduler running in no-daemon mode') if winutils.is_windows(): daemon = win_daemon.NoDaemon(daemonizable=freezer_scheduler) else: daemon = linux_daemon.NoDaemon(daemonizable=freezer_scheduler) else: if winutils.is_windows(): daemon = win_daemon.Daemon(daemonizable=freezer_scheduler, interval=int(CONF.interval), job_path=CONF.jobs_dir, insecure=CONF.insecure, concurrent_jobs=CONF.concurrent_jobs) else: daemon = linux_daemon.Daemon(daemonizable=freezer_scheduler) if CONF.action == 'start': daemon.start() elif CONF.action == 'stop': daemon.stop() elif CONF.action == 'reload': daemon.reload() elif CONF.action == 'status': daemon.status() # os.RETURN_CODES are only available to posix like systems, on windows # we need to translate the code to an actual number which is the equivalent return 0 # os.EX_OK
def test_create_dir(self): dir1 = '/tmp' dir2 = '/tmp/testnoexistent1234' dir3 = '~' assert utils.create_dir(dir1) is None assert utils.create_dir(dir2) is None os.rmdir(dir2) assert utils.create_dir(dir3) is None
def test_create_dir(self): dir1 = '/tmp' dir2 = '/tmp/testnoexistent1234' dir3 = '~' assert utils.create_dir(dir1) is None assert utils.create_dir(dir2) is None os.rmdir(dir2) assert utils.create_dir(dir3) is None
def do_job_download(client, args): freezer_utils.create_dir(args.jobs_dir, do_log=True) for doc in _job_list(client, args): fname = os.path.normpath('{0}/job_{1}.conf'. format(args.jobs_dir, doc['job_id'])) try: utils.save_doc_to_json_file(doc, fname) except Exception: print("Unable to write to file {0}".format(fname))
def prepare_logging(log_file='~/.freezer/freezer.log'): """ Creates log directory and log file if no log files provided :return: """ expanded_file_name = os.path.expanduser(log_file) expanded_dir_name = os.path.dirname(expanded_file_name) utils.create_dir(expanded_dir_name, do_log=False) return expanded_file_name
def do_job_download(client, args): freezer_utils.create_dir(args.jobs_dir, do_log=True) for doc in _job_list(client, args): fname = os.path.normpath('{0}/job_{1}.conf'.format( args.jobs_dir, doc['job_id'])) try: utils.save_doc_to_json_file(doc, fname) except Exception: print("Unable to write to file {0}".format(fname))
def prepare_logging(log_file='~/.freezer/freezer.log'): """ Creates log directory and log file if no log files provided :return: """ expanded_file_name = os.path.expanduser(log_file) expanded_dir_name = os.path.dirname(expanded_file_name) utils.create_dir(expanded_dir_name, do_log=False) return expanded_file_name
def prepare_logging(log_file=None): """ Creates log directory and log file if no log files provided :return: """ if not log_file: log_file = os.path.join(home, '.freezer', 'freezer.log') expanded_file_name = os.path.expanduser(log_file) expanded_dir_name = os.path.dirname(expanded_file_name) utils.create_dir(expanded_dir_name, do_log=False) return expanded_file_name
def prepare_logging(log_file=None): """ Creates log directory and log file if no log files provided :return: """ if not log_file: log_file = os.path.join(home, '.freezer', 'freezer.log') expanded_file_name = os.path.expanduser(log_file) expanded_dir_name = os.path.dirname(expanded_file_name) utils.create_dir(expanded_dir_name, do_log=False) return expanded_file_name
def start(self, log_file=None): """Initialize freezer-scheduler instance inside a windows service """ setup_logging(log_file) utils.create_dir(self.home) if self.insecure: os.environ['SERVICE_INSECURE'] = 'True' # send arguments info to the windows service os.environ['SERVICE_JOB_PATH'] = self.job_path os.environ['SERVICE_INTERVAL'] = str(self.interval) winutils.save_environment(self.home) print('Freezer Service is starting') win32serviceutil.StartService(self.service_name)
def start(self, log_file=None): """Initialize freezer-scheduler instance inside a windows service """ setup_logging(log_file) utils.create_dir(self.home) if self.insecure: os.environ['SERVICE_INSECURE'] = 'True' # send arguments info to the windows service os.environ['SERVICE_JOB_PATH'] = self.job_path os.environ['SERVICE_INTERVAL'] = str(self.interval) winutils.save_environment(self.home) print('Freezer Service is starting') win32serviceutil.StartService(self.service_name)
def configure_logging(file_name): expanded_file_name = os.path.expanduser(file_name) expanded_dir_name = os.path.dirname(expanded_file_name) utils.create_dir(expanded_dir_name, do_log=False) logging.basicConfig( filename=expanded_file_name, level=logging.INFO, format=('%(asctime)s %(name)s %(levelname)s %(message)s')) # filter out some annoying messages # not the best position for this code log_filter = NoLogFilter() logging.getLogger("apscheduler.scheduler").\ addFilter(log_filter) logging.getLogger("apscheduler.executors.default").\ addFilter(log_filter) logging.getLogger("requests.packages.urllib3.connectionpool").\ addFilter(log_filter) return expanded_file_name
def configure_logging(file_name): expanded_file_name = os.path.expanduser(file_name) expanded_dir_name = os.path.dirname(expanded_file_name) utils.create_dir(expanded_dir_name, do_log=False) logging.basicConfig( filename=expanded_file_name, level=logging.INFO, format=('%(asctime)s %(name)s %(levelname)s %(message)s')) # filter out some annoying messages # not the best position for this code log_filter = NoLogFilter() logging.getLogger("apscheduler.scheduler").\ addFilter(log_filter) logging.getLogger("apscheduler.executors.default").\ addFilter(log_filter) logging.getLogger("requests.packages.urllib3.connectionpool").\ addFilter(log_filter) return expanded_file_name
def download_meta_file(self, backup): """ Downloads meta_data to work_dir of previous backup. :type backup: freezer.storage.base.Backup :param backup: A backup or increment. Current backup is incremental, that means we should download tar_meta for detection new files and changes. If backup.tar_meta is false, raise Exception :return: """ utils.create_dir(self.work_dir) if backup.level == 0: return utils.path_join(self.work_dir, backup.tar()) meta_backup = backup.full_backup.increments[backup.level - 1] if not meta_backup.tar_meta: raise ValueError('Latest update have no tar_meta') to_path = utils.path_join(self.work_dir, meta_backup.tar()) if os.path.exists(to_path): os.remove(to_path) meta_backup.storage.get_file( meta_backup.storage.meta_file_abs_path(meta_backup), to_path) return to_path
def download_meta_file(self, backup): """ Downloads meta_data to work_dir of previous backup. :type backup: freezer.storage.base.Backup :param backup: A backup or increment. Current backup is incremental, that means we should download tar_meta for detection new files and changes. If backup.tar_meta is false, raise Exception :return: """ utils.create_dir(self.work_dir) if backup.level == 0: return utils.path_join(self.work_dir, backup.tar()) meta_backup = backup.full_backup.increments[backup.level - 1] if not meta_backup.tar_meta: raise ValueError('Latest update have no tar_meta') to_path = utils.path_join(self.work_dir, meta_backup.tar()) if os.path.exists(to_path): os.remove(to_path) meta_backup.storage.get_file( meta_backup.storage.meta_file_abs_path(meta_backup), to_path) return to_path
def create_dirs(self): tmpdir = tempfile.mkdtemp() if self.temp: backup_dir = tempfile.mkdtemp(dir=tmpdir, prefix=self.BACKUP_DIR_PREFIX) files_dir = tempfile.mkdtemp(dir=tmpdir, prefix=self.FILES_DIR_PREFIX) work_dir = tempfile.mkdtemp(dir=tmpdir, prefix=self.WORK_DIR_PREFIX) else: backup_dir = tmpdir + self.BACKUP_DIR_PREFIX files_dir = tmpdir + self.FILES_DIR_PREFIX work_dir = tmpdir + self.WORK_DIR_PREFIX utils.create_dir(backup_dir) utils.create_dir(work_dir) utils.create_dir(files_dir) self.create_content(files_dir) return backup_dir, files_dir, work_dir
def create_dirs(self): tmpdir = tempfile.mkdtemp() if self.temp: backup_dir = tempfile.mkdtemp( dir=tmpdir, prefix=self.BACKUP_DIR_PREFIX) files_dir = tempfile.mkdtemp( dir=tmpdir, prefix=self.FILES_DIR_PREFIX) work_dir = tempfile.mkdtemp( dir=tmpdir, prefix=self.WORK_DIR_PREFIX) else: backup_dir = tmpdir + self.BACKUP_DIR_PREFIX files_dir = tmpdir + self.FILES_DIR_PREFIX work_dir = tmpdir + self.WORK_DIR_PREFIX utils.create_dir(backup_dir) utils.create_dir(work_dir) utils.create_dir(files_dir) self.create_content(files_dir) return backup_dir, files_dir, work_dir
def lvm_snap(backup_opt_dict): """ Checks the provided parameters and create the lvm snapshot if requested The path_to_backup might be adjusted in case the user requested a lvm snapshot without specifying an exact path for the snapshot). The assumption in this case is that the user wants to use the lvm snapshot capability to backup the specified filesystem path, leaving out all the rest of the parameters which will guessed and set by freezer. :param backup_opt_dict: the configuration dict :return: True if the snapshot has been taken, False otherwise """ lvm_uuid = uuidutils.generate_uuid(dashed=False) if not backup_opt_dict.lvm_snapname: backup_opt_dict.lvm_snapname = \ "{0}_{1}".format(freezer_config.DEFAULT_LVM_SNAP_BASENAME, lvm_uuid) # adjust/check lvm parameters according to provided path_to_backup lvm_info = get_lvm_info(backup_opt_dict.path_to_backup) if not backup_opt_dict.lvm_volgroup: backup_opt_dict.lvm_volgroup = lvm_info['volgroup'] if not backup_opt_dict.lvm_srcvol: backup_opt_dict.lvm_srcvol = lvm_info['srcvol'] if not backup_opt_dict.lvm_dirmount: utils.create_dir(freezer_config.DEFAULT_LVM_MOUNT_BASEDIR) backup_opt_dict.lvm_dirmount = \ "{0}/mount_{1}".format(freezer_config.DEFAULT_LVM_MOUNT_BASEDIR, lvm_uuid) backup_opt_dict.path_to_backup = os.path.join(backup_opt_dict.lvm_dirmount, lvm_info['snap_path']) if not validate_lvm_params(backup_opt_dict): LOG.info('No LVM requested/configured') return False utils.create_dir(backup_opt_dict.lvm_dirmount) if '%' in backup_opt_dict.lvm_snapsize: lvm_size_option = "--extents" else: lvm_size_option = "--size" lvm_create_command = ( '{0} {1} {2} --snapshot --permission {3} ' '--name {4} {5}'.format( utils.find_executable('lvcreate'), lvm_size_option, backup_opt_dict.lvm_snapsize, ('r' if backup_opt_dict.lvm_snapperm == 'ro' else backup_opt_dict.lvm_snapperm), backup_opt_dict.lvm_snapname, backup_opt_dict.lvm_srcvol)) lvm_process = subprocess.Popen( lvm_create_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, executable=utils.find_executable('bash')) (lvm_out, lvm_err) = lvm_process.communicate() if lvm_process.returncode: raise Exception('lvm snapshot creation error: {0}'.format(lvm_err)) LOG.debug('{0}'.format(lvm_out)) LOG.warning('Logical volume "{0}" created'. format(backup_opt_dict.lvm_snapname)) # Guess the file system of the provided source volume and st mount # options accordingly filesys_type = get_vol_fs_type(backup_opt_dict.lvm_srcvol) mount_options = '-o {}'.format(backup_opt_dict.lvm_snapperm) if 'xfs' == filesys_type: mount_options = ' -onouuid ' # Mount the newly created snapshot to dir_mount abs_snap_name = '/dev/{0}/{1}'.format( backup_opt_dict.lvm_volgroup, backup_opt_dict.lvm_snapname) mount_command = '{0} {1} {2} {3}'.format( utils.find_executable('mount'), mount_options, abs_snap_name, backup_opt_dict.lvm_dirmount) mount_process = subprocess.Popen( mount_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, executable=utils.find_executable('bash')) mount_err = mount_process.communicate()[1] if 'already mounted' in mount_err: LOG.warning('Volume {0} already mounted on {1}\ '.format(abs_snap_name, backup_opt_dict.lvm_dirmount)) return True if mount_err: LOG.error("Snapshot mount error. Removing snapshot") lvm_snap_remove(backup_opt_dict) raise Exception('lvm snapshot mounting error: {0}'.format(mount_err)) else: LOG.warning( 'Volume {0} successfully mounted on {1}'.format( abs_snap_name, backup_opt_dict.lvm_dirmount)) return True
def lvm_snap(backup_opt_dict): """ Checks the provided parameters and create the lvm snapshot if requested The path_to_backup might be adjusted in case the user requested a lvm snapshot without specifying an exact path for the snapshot). The assumption in this case is that the user wants to use the lvm snapshot capability to backup the specified filesystem path, leaving out all the rest of the parameters which will guessed and set by freezer. :param backup_opt_dict: the configuration dict :return: True if the snapshot has been taken, False otherwise """ lvm_uuid = uuidutils.generate_uuid(dashed=False) if not backup_opt_dict.lvm_snapname: backup_opt_dict.lvm_snapname = \ "{0}_{1}".format(freezer_config.DEFAULT_LVM_SNAP_BASENAME, lvm_uuid) # adjust/check lvm parameters according to provided path_to_backup lvm_info = get_lvm_info(backup_opt_dict.path_to_backup) if not backup_opt_dict.lvm_volgroup: backup_opt_dict.lvm_volgroup = lvm_info['volgroup'] if not backup_opt_dict.lvm_srcvol: backup_opt_dict.lvm_srcvol = lvm_info['srcvol'] if not backup_opt_dict.lvm_dirmount: utils.create_dir(freezer_config.DEFAULT_LVM_MOUNT_BASEDIR) backup_opt_dict.lvm_dirmount = \ "{0}/mount_{1}".format(freezer_config.DEFAULT_LVM_MOUNT_BASEDIR, lvm_uuid) if not validate_lvm_params(backup_opt_dict): LOG.info('No LVM requested/configured') return False utils.create_dir(backup_opt_dict.lvm_dirmount) if '%' in backup_opt_dict.lvm_snapsize: lvm_size_option = "--extents" else: lvm_size_option = "--size" lvm_create_command = ( '{0} {1} {2} --snapshot --permission {3} ' '--name {4} {5}'.format( utils.find_executable('lvcreate'), lvm_size_option, backup_opt_dict.lvm_snapsize, ('r' if backup_opt_dict.lvm_snapperm == 'ro' else backup_opt_dict.lvm_snapperm), backup_opt_dict.lvm_snapname, backup_opt_dict.lvm_srcvol)) lvm_process = subprocess.Popen( lvm_create_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, executable=utils.find_executable('bash')) (lvm_out, lvm_err) = lvm_process.communicate() if lvm_process.returncode: raise Exception('lvm snapshot creation error: {0}'.format(lvm_err)) LOG.debug('{0}'.format(lvm_out)) LOG.warning('Logical volume "{0}" created'. format(backup_opt_dict.lvm_snapname)) # Guess the file system of the provided source volume and st mount # options accordingly filesys_type = get_vol_fs_type(backup_opt_dict.lvm_srcvol) mount_options = '-o {}'.format(backup_opt_dict.lvm_snapperm) if 'xfs' == filesys_type: mount_options = ' -onouuid ' # Mount the newly created snapshot to dir_mount abs_snap_name = '/dev/{0}/{1}'.format( backup_opt_dict.lvm_volgroup, backup_opt_dict.lvm_snapname) mount_command = '{0} {1} {2} {3}'.format( utils.find_executable('mount'), mount_options, abs_snap_name, backup_opt_dict.lvm_dirmount) mount_process = subprocess.Popen( mount_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, executable=utils.find_executable('bash')) mount_err = mount_process.communicate()[1] if 'already mounted' in mount_err: LOG.warning('Volume {0} already mounted on {1}\ '.format(abs_snap_name, backup_opt_dict.lvm_dirmount)) return True if mount_err: LOG.error("Snapshot mount error. Removing snapshot") lvm_snap_remove(backup_opt_dict) raise Exception('lvm snapshot mounting error: {0}'.format(mount_err)) else: LOG.warning( 'Volume {0} successfully mounted on {1}'.format( abs_snap_name, backup_opt_dict.lvm_dirmount)) # After snapshot is mounted, adjust path_to_backup according the mount # point and relative path of the snapshot volume. lvm_info = get_lvm_info(backup_opt_dict.path_to_backup) backup_opt_dict.path_to_backup = os.path.join(backup_opt_dict.lvm_dirmount, lvm_info['snap_path']) return True
def lvm_snap(backup_opt_dict): """ Checks the provided parameters and create the lvm snapshot if requested The path_to_backup might be adjusted in case the user requested a lvm snapshot without specifying an exact path for the snapshot (lvm_auto_snap). The assumption in this case is that the user wants to use the lvm snapshot capability to backup the specified filesystem path, leaving out all the rest of the parameters which will guessed and set by freezer. if a snapshot is requested using the --snapshot flag, but lvm_auto_snap is not provided, then path_to_backup is supposed to be the path to backup *before* any information about the snapshot is added and will be adjusted. :param backup_opt_dict: the configuration dict :return: True if the snapshot has been taken, False otherwise """ if backup_opt_dict.snapshot: if not backup_opt_dict.lvm_auto_snap: # 1) the provided path_to_backup has the meaning of # the lvm_auto_snap and is therefore copied into it # 2) the correct value of path_to_backup, which takes into # consideration the snapshot mount-point, is cleared # and will be calculated by freezer backup_opt_dict.lvm_auto_snap =\ backup_opt_dict.path_to_backup backup_opt_dict.path_to_backup = '' if not backup_opt_dict.lvm_snapname: backup_opt_dict.lvm_snapname = \ "{0}_{1}".format(freezer_config.DEFAULT_LVM_SNAP_BASENAME, uuid.uuid4().hex) if backup_opt_dict.lvm_auto_snap: # adjust/check lvm parameters according to provided lvm_auto_snap lvm_info = get_lvm_info(backup_opt_dict.lvm_auto_snap) if not backup_opt_dict.lvm_volgroup: backup_opt_dict.lvm_volgroup = lvm_info['volgroup'] if not backup_opt_dict.lvm_srcvol: backup_opt_dict.lvm_srcvol = lvm_info['srcvol'] if not backup_opt_dict.lvm_dirmount: backup_opt_dict.lvm_dirmount = \ "{0}_{1}".format(freezer_config.DEFAULT_LVM_MOUNT_BASENAME, uuid.uuid4().hex) path_to_backup = os.path.join(backup_opt_dict.lvm_dirmount, lvm_info['snap_path']) if backup_opt_dict.path_to_backup: # path_to_backup is user-provided, check if consistent if backup_opt_dict.path_to_backup != path_to_backup: raise Exception('Path to backup mismatch. ' 'provided: {0}, should be LVM-mounted: {1}'. format(backup_opt_dict.path_to_backup, path_to_backup)) else: # path_to_backup not provided: use the one calculated above backup_opt_dict.path_to_backup = path_to_backup if not validate_lvm_params(backup_opt_dict): logging.info('[*] No LVM requested/configured') return False utils.create_dir(backup_opt_dict.lvm_dirmount) lvm_create_command = ( '{0} --size {1} --snapshot --permission {2} ' '--name {3} {4}'.format( utils.find_executable('lvcreate'), backup_opt_dict.lvm_snapsize, ('r' if backup_opt_dict.lvm_snapperm == 'ro' else backup_opt_dict.lvm_snapperm), backup_opt_dict.lvm_snapname, backup_opt_dict.lvm_srcvol)) lvm_process = subprocess.Popen( lvm_create_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, executable=utils.find_executable('bash')) (lvm_out, lvm_err) = lvm_process.communicate() if lvm_process.returncode: raise Exception('lvm snapshot creation error: {0}'.format(lvm_err)) logging.debug('[*] {0}'.format(lvm_out)) logging.warning('[*] Logical volume "{0}" created'. format(backup_opt_dict.lvm_snapname)) # Guess the file system of the provided source volume and st mount # options accordingly filesys_type = get_vol_fs_type(backup_opt_dict.lvm_srcvol) mount_options = '-o {}'.format(backup_opt_dict.lvm_snapperm) if 'xfs' == filesys_type: mount_options = ' -onouuid ' # Mount the newly created snapshot to dir_mount abs_snap_name = '/dev/{0}/{1}'.format( backup_opt_dict.lvm_volgroup, backup_opt_dict.lvm_snapname) mount_command = '{0} {1} {2} {3}'.format( utils.find_executable('mount'), mount_options, abs_snap_name, backup_opt_dict.lvm_dirmount) mount_process = subprocess.Popen( mount_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, executable=utils.find_executable('bash')) mount_err = mount_process.communicate()[1] if 'already mounted' in mount_err: logging.warning('[*] Volume {0} already mounted on {1}\ '.format(abs_snap_name, backup_opt_dict.lvm_dirmount)) return True if mount_err: logging.error("[*] Snapshot mount error. Removing snapshot") lvm_snap_remove(backup_opt_dict) raise Exception('lvm snapshot mounting error: {0}'.format(mount_err)) else: logging.warning( '[*] Volume {0} succesfully mounted on {1}'.format( abs_snap_name, backup_opt_dict.lvm_dirmount)) return True