def main():
possible_actions = ['start', 'stop', 'status', 'reload']
arguments.parse_args(possible_actions)
arguments.setup_logging()
if CONF.action is None or CONF.action not in possible_actions:
CONF.print_help()
return 65 # os.EX_DATAERR
apiclient = None
if CONF.no_api is False:
try:
apiclient = client.Client(opts=CONF)
if CONF.client_id:
apiclient.client_id = CONF.client_id
except Exception as e:
LOG.error(e)
print(e)
sys.exit(1)
else:
if winutils.is_windows():
print("--no-api mode is not available on windows")
return 69 # os.EX_UNAVAILABLE
freezer_utils.create_dir(CONF.jobs_dir, do_log=False)
freezer_scheduler = FreezerScheduler(apiclient=apiclient,
interval=int(CONF.interval),
job_path=CONF.jobs_dir,
concurrent_jobs=CONF.concurrent_jobs)
if CONF.no_daemon:
print('Freezer Scheduler running in no-daemon mode')
LOG.debug('Freezer Scheduler running in no-daemon mode')
if winutils.is_windows():
daemon = win_daemon.NoDaemon(daemonizable=freezer_scheduler)
else:
daemon = linux_daemon.NoDaemon(daemonizable=freezer_scheduler)
else:
if winutils.is_windows():
daemon = win_daemon.Daemon(daemonizable=freezer_scheduler,
interval=int(CONF.interval),
job_path=CONF.jobs_dir,
insecure=CONF.insecure,
concurrent_jobs=CONF.concurrent_jobs)
else:
daemon = linux_daemon.Daemon(daemonizable=freezer_scheduler)
if CONF.action == 'start':
daemon.start()
elif CONF.action == 'stop':
daemon.stop()
elif CONF.action == 'reload':
daemon.reload()
elif CONF.action == 'status':
daemon.status()
# os.RETURN_CODES are only available to posix like systems, on windows
# we need to translate the code to an actual number which is the equivalent
return 0 # os.EX_OK
def snapshot_remove(backup_opt_dict, shadow, windows_volume):
if winutils.is_windows():
# Delete the shadow copy after the backup
vss.vss_delete_shadow_copy(shadow, windows_volume)
else:
# Unmount and remove lvm snapshot volume
lvm.lvm_snap_remove(backup_opt_dict)
def snapshot_remove(backup_opt_dict, shadow, windows_volume):
if winutils.is_windows():
# Delete the shadow copy after the backup
vss.vss_delete_shadow_copy(shadow, windows_volume)
else:
# Unmount and remove lvm snapshot volume
lvm.lvm_snap_remove(backup_opt_dict)
def restore_level(self, restore_path, read_pipe):
"""
Restore the provided file into backup_opt_dict.restore_abs_path
Decrypt the file if backup_opt_dict.encrypt_pass_file key is provided
"""
tar_command = tar_builders.TarCommandRestoreBuilder(
restore_path, self.compression_algo, self.is_windows)
if self.encrypt_pass_file:
tar_command.set_encryption(self.encrypt_pass_file)
if self.dry_run:
tar_command.set_dry_run()
command = tar_command.build()
if winutils.is_windows():
# on windows, chdir to restore path.
os.chdir(restore_path)
tar_process = subprocess.Popen(
command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
# Start loop reading the pipe and pass the data to the tar std input.
# If EOFError exception is raised, the loop end the std err will be
# checked for errors.
try:
while True:
tar_process.stdin.write(read_pipe.recv_bytes())
except EOFError:
logging.info('[*] Pipe closed as EOF reached. '
'Data transmitted successfully')
self.check_process_output(tar_process)
def restore_level(self, restore_resource, read_pipe, backup, except_queue):
"""
Restore the provided file into backup_opt_dict.restore_abs_path
Decrypt the file if backup_opt_dict.encrypt_pass_file key is provided
:param restore_path:
:param read_pipe:
:type backup: freezer.storage.base.Backup
:param backup:
"""
try:
metadata = backup.metadata()
if (not self.encrypt_pass_file
and metadata.get("encryption", False)):
raise Exception("Cannot restore encrypted backup without key")
tar_command = tar_builders.TarCommandRestoreBuilder(
restore_resource,
metadata.get('compression', self.compression_algo),
self.is_windows)
if self.encrypt_pass_file:
tar_command.set_encryption(self.encrypt_pass_file)
if self.dry_run:
tar_command.set_dry_run()
command = tar_command.build()
if winutils.is_windows():
# on windows, chdir to restore path.
os.chdir(restore_resource)
tar_process = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
executable='/bin/bash')
# Start loop reading the pipe and pass the data to the tar
# std input. If EOFError exception is raised, the loop end
# the std err will be checked for errors.
try:
while True:
tar_process.stdin.write(read_pipe.recv_bytes())
except EOFError:
LOG.info('Pipe closed as EOF reached. '
'Data transmitted successfully')
finally:
self.check_process_output(tar_process, 'Restore')
except Exception as e:
LOG.exception(e)
except_queue.put(e)
raise
def restore_level(self, restore_resource, read_pipe, backup, except_queue):
"""
Restore the provided file into backup_opt_dict.restore_abs_path
Decrypt the file if backup_opt_dict.encrypt_pass_file key is provided
:param restore_path:
:param read_pipe:
:type backup: freezer.storage.base.Backup
:param backup:
"""
try:
metadata = backup.metadata()
if (not self.encrypt_pass_file and
metadata.get("encryption", False)):
raise Exception("Cannot restore encrypted backup without key")
tar_command = tar_builders.TarCommandRestoreBuilder(
restore_resource,
metadata.get('compression', self.compression_algo),
self.is_windows)
if self.encrypt_pass_file:
tar_command.set_encryption(self.encrypt_pass_file)
if self.dry_run:
tar_command.set_dry_run()
command = tar_command.build()
if winutils.is_windows():
# on windows, chdir to restore path.
os.chdir(restore_resource)
tar_process = subprocess.Popen(
command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True, executable='/bin/bash')
# Start loop reading the pipe and pass the data to the tar
# std input. If EOFError exception is raised, the loop end
# the std err will be checked for errors.
try:
while True:
tar_process.stdin.write(read_pipe.recv_bytes())
except EOFError:
LOG.info('Pipe closed as EOF reached. '
'Data transmitted successfully')
finally:
self.check_process_output(tar_process, 'Restore')
except Exception as e:
LOG.exception(e)
except_queue.put(e)
raise
def __init__(self, **kwargs):
self.compression_algo = kwargs.get('compression')
self.encrypt_pass_file = kwargs.get('encrypt_key', None)
self.dereference_symlink = kwargs.get('symlinks')
self.exclude = kwargs.get('exclude')
self.storage = kwargs.get('storage')
self.is_windows = winutils.is_windows()
self.dry_run = kwargs.get('dry_run', False)
self.max_segment_size = kwargs.get('max_segment_size')
self.rsync_block_size = kwargs.get('rsync_block_size')
self.fixed_blocks = 0
self.modified_blocks = 0
super(Rsyncv2Engine, self).__init__(storage=kwargs.get('storage'))
def tar_path():
"""This function returns tar binary path"""
from freezer.utils import winutils
if winutils.is_windows():
path_to_binaries = os.path.dirname(os.path.abspath(__file__))
return '{0}\\bin\\tar.exe'.format(path_to_binaries)
tar = (get_executable_path('gnutar') or get_executable_path('gtar')
or get_executable_path('tar'))
if not tar:
raise Exception('Please install gnu tar (gtar) as it is a '
'mandatory requirement to use freezer.')
return tar
def tar_path():
"""This function returns tar binary path"""
from freezer.utils import winutils
if winutils.is_windows():
path_to_binaries = os.path.dirname(os.path.abspath(__file__))
return '{0}\\bin\\tar.exe'.format(path_to_binaries)
tar = (get_executable_path('gnutar') or get_executable_path('gtar') or
get_executable_path('tar'))
if not tar:
raise Exception('Please install gnu tar (gtar) as it is a '
'mandatory requirement to use freezer.')
return tar
def __init__(self, **kwargs):
self.compression_algo = kwargs.get('compression')
self.encrypt_pass_file = kwargs.get('encrypt_key', None)
self.dereference_symlink = kwargs.get('symlinks')
self.exclude = kwargs.get('exclude')
self.storage = kwargs.get('storage')
self.is_windows = winutils.is_windows()
self.dry_run = kwargs.get('dry_run', False)
self.max_segment_size = kwargs.get('max_segment_size')
self.rsync_block_size = kwargs.get('rsync_block_size')
self.fixed_blocks = 0
self.modified_blocks = 0
super(Rsyncv2Engine, self).__init__(storage=kwargs.get('storage'))
def __init__(self, storage, **kwargs):
super(OsbrickEngine, self).__init__(storage=storage)
self.client = client_manager.get_client_manager(CONF)
self.cinder = self.client.create_cinder()
self.volume_info = None
self.compression_algo = kwargs.get('compression')
self.encrypt_pass_file = kwargs.get('encrypt_key')
self.dereference_symlink = kwargs.get('symlinks')
self.exclude = kwargs.get('exclude')
self.storage = storage
self.is_windows = winutils.is_windows()
self.dry_run = kwargs.get('dry_run', False)
self.max_segment_size = kwargs.get('max_segment_size')
def __init__(self, storage, **kwargs):
super(OsbrickEngine, self).__init__(storage=storage)
self.client = client_manager.get_client_manager(CONF)
self.cinder = self.client.create_cinder()
self.volume_info = None
self.compression_algo = kwargs.get('compression')
self.encrypt_pass_file = kwargs.get('encrypt_key')
self.dereference_symlink = kwargs.get('symlinks')
self.exclude = kwargs.get('exclude')
self.storage = storage
self.is_windows = winutils.is_windows()
self.dry_run = kwargs.get('dry_run', False)
self.max_segment_size = kwargs.get('max_segment_size')
def __init__(
self, compression, symlinks, exclude, storage,
max_segment_size, encrypt_key=None,
dry_run=False):
self.compression_algo = compression
self.encrypt_pass_file = encrypt_key
self.dereference_symlink = symlinks
self.exclude = exclude
self.storage = storage
self.is_windows = winutils.is_windows()
self.dry_run = dry_run
self.max_segment_size = max_segment_size
# Compression and encryption objects
self.compressor = None
self.cipher = None
super(RsyncEngine, self).__init__(storage=storage)
def __init__(
self, compression, symlinks, exclude, storage,
max_segment_size, encrypt_key=None,
dry_run=False, **kwargs):
self.compression_algo = compression
self.encrypt_pass_file = encrypt_key
self.dereference_symlink = symlinks
self.exclude = exclude
self.storage = storage
self.is_windows = winutils.is_windows()
self.dry_run = dry_run
self.max_segment_size = max_segment_size
# Compression and encryption objects
self.compressor = None
self.cipher = None
super(RsyncEngine, self).__init__(storage=storage)
def __init__(
self, compression, symlinks, exclude, storage,
max_segment_size, encrypt_key=None,
dry_run=False, **kwargs):
"""
:type storage: freezer.storage.base.Storage
:return:
"""
self.compression_algo = compression
self.encrypt_pass_file = encrypt_key
self.dereference_symlink = symlinks
self.exclude = exclude
self.storage = storage
self.is_windows = winutils.is_windows()
self.dry_run = dry_run
self.max_segment_size = max_segment_size
super(TarEngine, self).__init__(storage=storage)
def __init__(
self, compression, symlinks, exclude, storage,
max_segment_size, encrypt_key=None,
dry_run=False):
"""
:type storage: freezer.storage.base.Storage
:return:
"""
self.compression_algo = compression
self.encrypt_pass_file = encrypt_key
self.dereference_symlink = symlinks
self.exclude = exclude
self.storage = storage
self.is_windows = winutils.is_windows()
self.dry_run = dry_run
self.max_segment_size = max_segment_size
super(TarEngine, self).__init__(storage=storage)
def get_executable_path(binary):
"""
This function returns the executable path of a given binary
if it is found in the system.
:param binary:
:type binary: str
:rtype: str
:return: Absolute Path to the executable file
"""
from freezer.utils import winutils
if winutils.is_windows():
path_to_binaries = os.path.dirname(os.path.abspath(__file__))
return '{0}\\bin\\{1}.exe'.format(path_to_binaries, binary)
elif is_bsd():
return (distspawn.find_executable(binary) or
distspawn.find_executable(binary, path=':'.join(sys.path)))
else:
return distspawn.find_executable(binary)
def get_executable_path(binary):
"""
This function returns the executable path of a given binary
if it is found in the system.
:param binary:
:type binary: str
:rtype: str
:return: Absolute Path to the executable file
"""
from freezer.utils import winutils
if winutils.is_windows():
path_to_binaries = os.path.dirname(os.path.abspath(__file__))
return '{0}\\bin\\{1}.exe'.format(path_to_binaries, binary)
elif is_bsd():
return (distspawn.find_executable(binary) or
distspawn.find_executable(binary, path=':'.join(sys.path)))
else:
return distspawn.find_executable(binary)
def restore_level(self, restore_path, read_pipe, backup):
"""
Restore the provided file into backup_opt_dict.restore_abs_path
Decrypt the file if backup_opt_dict.encrypt_pass_file key is provided
"""
metadata = backup.metadata()
if not self.encrypt_pass_file and metadata.get("encryption", False):
raise Exception("Cannot restore encrypted backup without key")
tar_command = tar_builders.TarCommandRestoreBuilder(
restore_path, metadata.get('compression', self.compression_algo),
self.is_windows)
if self.encrypt_pass_file:
tar_command.set_encryption(self.encrypt_pass_file)
if self.dry_run:
tar_command.set_dry_run()
command = tar_command.build()
if winutils.is_windows():
# on windows, chdir to restore path.
os.chdir(restore_path)
tar_process = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
# Start loop reading the pipe and pass the data to the tar std input.
# If EOFError exception is raised, the loop end the std err will be
# checked for errors.
try:
while True:
tar_process.stdin.write(read_pipe.recv_bytes())
except EOFError:
logging.info('[*] Pipe closed as EOF reached. '
'Data transmitted successfully')
self.check_process_output(tar_process)
def snapshot_create(backup_opt_dict):
"""
Calls the code to take fs snapshots, depending on the platform
:param backup_opt_dict:
:return: boolean value, True if snapshot has been taken, false otherwise
"""
if winutils.is_windows():
if backup_opt_dict.snapshot:
# Create a shadow copy.
backup_opt_dict.shadow_path, backup_opt_dict.shadow = \
vss.vss_create_shadow_copy(backup_opt_dict.windows_volume)
backup_opt_dict.path_to_backup = winutils.use_shadow(
backup_opt_dict.path_to_backup,
backup_opt_dict.windows_volume)
return True
return False
else:
return lvm.lvm_snap(backup_opt_dict)
def snapshot_create(backup_opt_dict):
"""
Calls the code to take fs snapshots, depending on the platform
:param backup_opt_dict:
:return: boolean value, True if snapshot has been taken, false otherwise
"""
if not backup_opt_dict.snapshot:
return False
if winutils.is_windows():
if backup_opt_dict.snapshot:
# Create a shadow copy.
backup_opt_dict.shadow_path, backup_opt_dict.shadow = \
vss.vss_create_shadow_copy(backup_opt_dict.windows_volume)
backup_opt_dict.path_to_backup = winutils.use_shadow(
backup_opt_dict.path_to_backup, backup_opt_dict.windows_volume)
return True
return False
else:
return lvm.lvm_snap(backup_opt_dict)
def get_backup_args():
defaults = DEFAULT_PARAMS.copy()
class FreezerConfig(object):
def __init__(self, args):
self.__dict__.update(args)
cli_options = dict([(x, y) for x, y in CONF.iteritems() if y is not None])
defaults.update(cli_options)
conf = None
if CONF.get('config'):
conf = freezer_config.Config.parse(CONF.get('config'))
defaults.update(conf.default)
# TODO: restore_from_host is deprecated and to be removed
defaults['hostname'] = conf.default.get('hostname') or \
conf.default.get('restore_from_host')
# override default oslo values
levels = {
'all': log.NOTSET,
'debug': log.DEBUG,
'warn': log.WARN,
'info': log.INFO,
'error': log.ERROR,
'critical': log.CRITICAL
}
if not CONF.get('log_file'):
CONF.set_override('log_file', levels.get(defaults['log_file'],
log.NOTSET))
CONF.set_override('default_log_levels', defaults['log_level'])
if not CONF.get('log_file'):
log_file = None
for file_name in ['/var/log/freezer.log', '~/.freezer/freezer.log']:
try:
log_file = prepare_logging(file_name)
except IOError:
pass
if log_file:
CONF.set_default('log_file', log_file)
else:
LOG.warn("log file cannot be created. Freezer will proceed with "
"default stdout and stderr")
backup_args = FreezerConfig(defaults)
# Set default working directory to ~/.freezer. If the directory
# does not exists it is created
work_dir = os.path.join(home, '.freezer')
backup_args.__dict__['work_dir'] = work_dir
if not os.path.exists(work_dir):
try:
os.makedirs(work_dir)
except (OSError, IOError) as err_msg:
# This avoids freezer-agent to crash if it can't write to
# ~/.freezer, which may happen on some env (for me,
# it happens in Jenkins, as freezer-agent can't write to
# /var/lib/jenkins).
print(encodeutils.safe_decode('{}'.format(err_msg)),
file=sys.stderr)
# If hostname is not set, hostname of the current node will be used
if not backup_args.hostname:
backup_args.__dict__['hostname'] = socket.gethostname()
# If we have provided --proxy then overwrite the system HTTP_PROXY and
# HTTPS_PROXY
if backup_args.proxy:
utils.alter_proxy(backup_args.proxy)
# MySQLdb object
backup_args.__dict__['mysql_db_inst'] = ''
backup_args.__dict__['storages'] = None
if conf and conf.storages:
backup_args.__dict__['storages'] = conf.storages
# Windows volume
backup_args.__dict__['shadow'] = ''
backup_args.__dict__['shadow_path'] = ''
backup_args.__dict__['file_name'] = ''
if winutils.is_windows():
if backup_args.path_to_backup:
backup_args.__dict__['windows_volume'] = \
backup_args.path_to_backup[:3]
# todo(enugaev) move it to new command line param backup_media
if backup_args.lvm_auto_snap:
raise Exception('lvm-auto-snap is deprecated. '
'Please use --snapshot instead')
backup_media = 'fs'
if backup_args.cinder_vol_id:
backup_media = 'cinder'
elif backup_args.cindernative_vol_id:
backup_media = 'cindernative'
elif backup_args.nova_inst_id:
backup_media = 'nova'
backup_args.__dict__['backup_media'] = backup_media
backup_args.__dict__['time_stamp'] = None
if backup_args.upload_limit != -1 or backup_args.download_limit != -1 and \
not winutils.is_windows():
# handle --config option with tmp config file
if backup_args.config:
conf_file = NamedTemporaryFile(prefix='freezer_job_', delete=False)
# remove the limits from the new file
if 'upload_limit' in conf.default:
conf.default.pop('upload_limit')
elif 'download_limit' in conf.default:
conf.default.pop('download_limit')
utils.save_config_to_file(conf.default, conf_file,
'default')
# replace the original file with the tmp one
conf_index = sys.argv.index('--config') + 1
sys.argv[conf_index] = conf_file.name
# if limits provided from cli remove it !
if '--upload-limit' in sys.argv:
index = sys.argv.index('--upload-limit')
sys.argv.pop(index)
sys.argv.pop(index)
if '--download-limit' in sys.argv:
index = sys.argv.index('--download-limit')
sys.argv.pop(index)
sys.argv.pop(index)
# locate trickle
trickle_executable = distspawn.find_executable('trickle')
if trickle_executable is None:
trickle_executable = distspawn.find_executable(
'trickle', path=":".join(sys.path))
if trickle_executable is None:
trickle_executable = distspawn.find_executable(
'trickle', path=":".join(os.environ.get('PATH')))
if trickle_executable:
LOG.info("[*] Info: Starting trickle ...")
trickle_command = '{0} -d {1} -u {2} '.\
format(trickle_executable,
getattr(backup_args, 'download_limit') or -1,
getattr(backup_args, 'upload_limit') or -1)
backup_args.__dict__['trickle_command'] = trickle_command
if backup_args.config:
backup_args.__dict__['tmp_file'] = conf_file.name
# maintain env variable not to get into infinite loop
if "tricklecount" in os.environ:
tricklecount = int(os.environ.get("tricklecount", 1))
tricklecount += 1
os.environ["tricklecount"] = str(tricklecount)
else:
os.environ["tricklecount"] = str(1)
else:
LOG.warn("[*] Trickle not found. Switching to normal mode without "
"limiting bandwidth")
if backup_args.config:
# remove index tmp_file from backup arguments dict
backup_args.__dict__.pop('tmp_file')
utils.delete_file(conf_file.name)
return backup_args
def freezer_main(backup_args):
"""Freezer main loop for job execution.
"""
if not backup_args.quiet:
LOG.info('log file at {0}'.format(CONF.get('log_file')))
if backup_args.max_priority:
utils.set_max_process_priority()
bandwidth.monkeypatch_socket_bandwidth(backup_args)
backup_args.__dict__['hostname_backup_name'] = "{0}_{1}".format(
backup_args.hostname, backup_args.backup_name)
validator.validate(backup_args)
work_dir = backup_args.work_dir
os_identity = backup_args.os_identity_api_version
max_segment_size = backup_args.max_segment_size
if backup_args.storages:
storage = multiple.MultipleStorage(
work_dir,
[storage_from_dict(x, work_dir, max_segment_size, os_identity)
for x in backup_args.storages])
else:
storage = storage_from_dict(backup_args.__dict__, work_dir,
max_segment_size, os_identity)
backup_args.__dict__['engine'] = tar_engine.TarBackupEngine(
backup_args.compression,
backup_args.dereference_symlink,
backup_args.exclude,
storage,
winutils.is_windows(),
backup_args.encrypt_pass_file,
backup_args.dry_run)
if hasattr(backup_args, 'trickle_command'):
if "tricklecount" in os.environ:
if int(os.environ.get("tricklecount")) > 1:
LOG.critical("[*] Trickle seems to be not working, Switching "
"to normal mode ")
run_job(backup_args, storage)
freezer_command = '{0} {1}'.format(backup_args.trickle_command,
' '.join(sys.argv))
process = subprocess.Popen(freezer_command.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=os.environ.copy())
while process.poll() is None:
print(process.stdout.readline().rstrip())
output, error = process.communicate()
if process.returncode:
LOG.error("[*] Trickle Error: {0}".format(error))
LOG.critical("[*] Switching to work without trickle ...")
run_job(backup_args, storage)
else:
run_job(backup_args, storage)
def test_is_windows(self):
fake_os = commons.Os()
os.name = fake_os
assert winutils.is_windows() is False
def get_backup_args():
defaults = DEFAULT_PARAMS.copy()
class FreezerConfig(object):
def __init__(self, args):
self.__dict__.update(args)
cli_options = dict([(x, y) for x, y in CONF.iteritems() if y is not None])
defaults.update(cli_options)
conf = None
if CONF.get('config'):
conf = freezer_config.Config.parse(CONF.get('config'))
# force log_config_append to always exists in defaults even if not
# provided.
defaults['log_config_append'] = None
defaults.update(conf.default)
if defaults['log_file']:
CONF.set_override('log_file', defaults['log_file'])
CONF.set_override('default_log_levels', _DEFAULT_LOG_LEVELS)
if not CONF.get('log_file'):
log_file = None
for file_name in [
'/var/log/freezer-agent/freezer-agent.log',
'/var/log/freezer.log'
]:
try:
log_file = prepare_logging(file_name)
except IOError:
pass
if not log_file:
# Set default working directory to ~/.freezer. If the directory
# does not exists it is created
work_dir = os.path.join(home, '.freezer')
if not os.path.exists(work_dir):
try:
os.makedirs(work_dir)
log_file = prepare_logging(
os.path.join(work_dir, 'freezer.log'))
except (OSError, IOError) as err_msg:
# This avoids freezer-agent to crash if it can't write to
# ~/.freezer, which may happen on some env (for me,
# it happens in Jenkins, as freezer-agent can't write to
# /var/lib/jenkins).
print(encodeutils.safe_decode('{}'.format(err_msg)),
file=sys.stderr)
if log_file:
CONF.set_default('log_file', log_file)
else:
LOG.warning("log file cannot be created. Freezer will proceed with"
" default stdout and stderr")
backup_args = FreezerConfig(defaults)
if CONF.get('config'):
backup_args.__dict__['config'] = CONF.get('config')
# Set default working directory to ~/.freezer. If the directory
# does not exists it is created
work_dir = os.path.join(home, '.freezer')
backup_args.__dict__['work_dir'] = work_dir
if not os.path.exists(work_dir):
try:
os.makedirs(work_dir)
except (OSError, IOError) as err_msg:
# This avoids freezer-agent to crash if it can't write to
# ~/.freezer, which may happen on some env (for me,
# it happens in Jenkins, as freezer-agent can't write to
# /var/lib/jenkins).
print(encodeutils.safe_decode('{}'.format(err_msg)),
file=sys.stderr)
# If hostname is not set, hostname of the current node will be used
if not backup_args.hostname:
backup_args.__dict__['hostname'] = socket.gethostname()
# If we have provided --proxy then overwrite the system HTTP_PROXY and
# HTTPS_PROXY
if backup_args.proxy:
utils.alter_proxy(backup_args.proxy)
# MySQLdb object
backup_args.__dict__['mysql_db_inst'] = ''
backup_args.__dict__['storages'] = None
if conf and conf.storages:
backup_args.__dict__['storages'] = conf.storages
# Windows volume
backup_args.__dict__['shadow'] = ''
backup_args.__dict__['shadow_path'] = ''
backup_args.__dict__['file_name'] = ''
if winutils.is_windows():
if backup_args.path_to_backup:
backup_args.__dict__['windows_volume'] = \
backup_args.path_to_backup[:3]
backup_media = 'fs'
if backup_args.cinder_vol_id:
backup_media = 'cinder'
elif backup_args.cindernative_vol_id or backup_args.cindernative_backup_id:
backup_media = 'cindernative'
elif backup_args.engine_name == 'nova' and (backup_args.project_id
or backup_args.nova_inst_id):
backup_media = 'nova'
elif backup_args.cinderbrick_vol_id:
backup_media = 'cinderbrick'
backup_args.__dict__['backup_media'] = backup_media
backup_args.__dict__['time_stamp'] = None
if backup_args.upload_limit != -1 or backup_args.download_limit != -1 and \
not winutils.is_windows():
# handle --config option with tmp config file
if backup_args.config:
conf_file = tempfile.NamedTemporaryFile(prefix='freezer_job_',
delete=False)
# remove the limits from the new file
if 'upload_limit' in conf.default:
conf.default.pop('upload_limit')
elif 'download_limit' in conf.default:
conf.default.pop('download_limit')
utils.save_config_to_file(conf.default, conf_file, 'default')
# replace the original file with the tmp one
conf_index = sys.argv.index('--config') + 1
sys.argv[conf_index] = conf_file.name
# if limits provided from cli remove it !
if '--upload-limit' in sys.argv:
index = sys.argv.index('--upload-limit')
sys.argv.pop(index)
sys.argv.pop(index)
if '--download-limit' in sys.argv:
index = sys.argv.index('--download-limit')
sys.argv.pop(index)
sys.argv.pop(index)
# locate trickle
trickle_executable = distspawn.find_executable('trickle')
if trickle_executable is None:
trickle_executable = distspawn.find_executable('trickle',
path=":".join(
sys.path))
if trickle_executable is None:
trickle_executable = distspawn.find_executable(
'trickle', path=":".join(os.environ.get('PATH')))
if trickle_executable:
LOG.info("Info: Starting trickle ...")
trickle_command = '{0} -d {1} -u {2} '.\
format(trickle_executable,
getattr(backup_args, 'download_limit') or -1,
getattr(backup_args, 'upload_limit') or -1)
backup_args.__dict__['trickle_command'] = trickle_command
if backup_args.config:
backup_args.__dict__['tmp_file'] = conf_file.name
# maintain env variable not to get into infinite loop
if "tricklecount" in os.environ:
tricklecount = int(os.environ.get("tricklecount", 1))
tricklecount += 1
os.environ["tricklecount"] = str(tricklecount)
else:
os.environ["tricklecount"] = str(1)
else:
LOG.warning("Trickle not found. Switching to normal mode without "
"limiting bandwidth")
if backup_args.config:
# remove index tmp_file from backup arguments dict
utils.delete_file(conf_file.name)
return backup_args
def openssl_path():
from freezer.utils import winutils
if winutils.is_windows():
return 'openssl'
else:
return find_executable('openssl')
def main():
doers = _get_doers(shell)
doers.update(_get_doers(utils))
possible_actions = doers.keys() + ['start', 'stop', 'status']
arguments.parse_args(possible_actions)
arguments.setup_logging()
if CONF.action is None:
CONF.print_help()
return 65 # os.EX_DATAERR
apiclient = None
insecure = False
if CONF.insecure:
insecure = True
if CONF.no_api is False:
try:
apiclient = client.Client(opts=CONF, insecure=insecure)
if CONF.client_id:
apiclient.client_id = CONF.client_id
except Exception as e:
LOG.error(e)
print(e)
sys.exit(1)
else:
if winutils.is_windows():
print("--no-api mode is not available on windows")
return 69 # os.EX_UNAVAILABLE
if CONF.action in doers:
try:
return doers[CONF.action](apiclient, CONF)
except Exception as e:
LOG.error(e)
print ('ERROR {0}'.format(e))
return 70 # os.EX_SOFTWARE
freezer_scheduler = FreezerScheduler(apiclient=apiclient,
interval=int(CONF.interval),
job_path=CONF.jobs_dir)
if CONF.no_daemon:
print ('Freezer Scheduler running in no-daemon mode')
LOG.debug('Freezer Scheduler running in no-daemon mode')
daemon = NoDaemon(daemonizable=freezer_scheduler)
else:
if winutils.is_windows():
daemon = Daemon(daemonizable=freezer_scheduler,
interval=int(CONF.interval),
job_path=CONF.jobs_dir,
insecure=CONF.insecure)
else:
daemon = Daemon(daemonizable=freezer_scheduler)
if CONF.action == 'start':
daemon.start(log_file=CONF.log_file)
elif CONF.action == 'stop':
daemon.stop()
elif CONF.action == 'reload':
daemon.reload()
elif CONF.action == 'status':
daemon.status()
# os.RETURN_CODES are only available to posix like systems, on windows
# we need to translate the code to an actual number which is the equivalent
return 0 # os.EX_OK
def get_backup_args():
defaults = DEFAULT_PARAMS.copy()
class FreezerConfig(object):
def __init__(self, args):
self.__dict__.update(args)
cli_options = dict([(x, y) for x, y in CONF.iteritems() if y is not None])
defaults.update(cli_options)
conf = None
if CONF.get('config'):
conf = freezer_config.Config.parse(CONF.get('config'))
# force log_config_append to always exists in defaults even if not
# provided.
defaults['log_config_append'] = None
defaults.update(conf.default)
for config_key in conf.default.keys():
try:
CONF.get(config_key)
CONF.set_override(config_key, conf.default[config_key])
except NoSuchOptError:
LOG.debug('No such opt, {0}, so set it'.format(config_key))
setattr(CONF, config_key, conf.default[config_key])
except KeyError:
real_opt, real_group = CONF._find_deprecated_opts(config_key)
if '-' in real_opt:
real_opt = real_opt.replace('-', '_')
CONF.set_override(real_opt, conf.default[real_opt])
if defaults['log_file']:
CONF.set_override('log_file', defaults['log_file'])
CONF.set_override('default_log_levels', _DEFAULT_LOG_LEVELS)
if not CONF.get('log_file'):
log_file = None
file_name = '/var/log/freezer-agent/freezer-agent.log'
try:
log_file = prepare_logging(file_name)
except IOError:
pass
if not log_file:
# Set default working directory to ~/.freezer. If the directory
# does not exists it is created
try:
log_file = prepare_logging()
except (OSError, IOError) as err_msg:
# This avoids freezer-agent to crash if it can't write to
# ~/.freezer, which may happen on some env (for me,
# it happens in Jenkins, as freezer-agent can't write to
# /var/lib/jenkins).
print(encodeutils.safe_decode('{}'.format(err_msg)),
file=sys.stderr)
if log_file:
CONF.set_default('log_file', log_file)
else:
LOG.warning("log file cannot be created. Freezer will proceed with"
" default stdout and stderr")
backup_args = FreezerConfig(defaults)
if CONF.get('config'):
backup_args.__dict__['config'] = CONF.get('config')
# Set default working directory to ~/.freezer. If the directory
# does not exists it is created
work_dir = os.path.join(home, '.freezer')
backup_args.__dict__['work_dir'] = work_dir
if not os.path.exists(work_dir):
try:
os.makedirs(work_dir)
except (OSError, IOError) as err_msg:
# This avoids freezer-agent to crash if it can't write to
# ~/.freezer, which may happen on some env (for me,
# it happens in Jenkins, as freezer-agent can't write to
# /var/lib/jenkins).
print(encodeutils.safe_decode('{}'.format(err_msg)),
file=sys.stderr)
# If hostname is not set, hostname of the current node will be used
if not backup_args.hostname:
backup_args.__dict__['hostname'] = socket.gethostname()
# If we have provided --proxy then overwrite the system HTTP_PROXY and
# HTTPS_PROXY
if backup_args.proxy:
utils.alter_proxy(backup_args.proxy)
# MySQLdb object
backup_args.__dict__['mysql_db_inst'] = ''
backup_args.__dict__['storages'] = None
if conf and conf.storages:
backup_args.__dict__['storages'] = conf.storages
# Windows volume
backup_args.__dict__['shadow'] = ''
backup_args.__dict__['shadow_path'] = ''
backup_args.__dict__['file_name'] = ''
if winutils.is_windows():
if backup_args.path_to_backup:
backup_args.__dict__['windows_volume'] = \
backup_args.path_to_backup[:3]
backup_media = 'fs'
if backup_args.cinder_vol_id or backup_args.cinder_vol_name:
backup_media = 'cinder'
elif backup_args.cindernative_vol_id or backup_args.cindernative_backup_id:
backup_media = 'cindernative'
elif backup_args.engine_name == 'nova' and (backup_args.project_id or
backup_args.nova_inst_id or
backup_args.nova_inst_name):
backup_media = 'nova'
elif backup_args.cinderbrick_vol_id:
backup_media = 'cinderbrick'
backup_args.__dict__['backup_media'] = backup_media
backup_args.__dict__['time_stamp'] = None
if backup_args.upload_limit != -1 or backup_args.download_limit != -1 and \
not winutils.is_windows():
# handle --config option with tmp config file
if backup_args.config:
conf_file = tempfile.NamedTemporaryFile(prefix='freezer_job_',
delete=False)
# remove the limits from the new file
if 'upload_limit' in conf.default:
conf.default.pop('upload_limit')
elif 'download_limit' in conf.default:
conf.default.pop('download_limit')
utils.save_config_to_file(conf.default, conf_file,
'default')
# replace the original file with the tmp one
conf_index = sys.argv.index('--config') + 1
sys.argv[conf_index] = conf_file.name
# if limits provided from cli remove it !
if '--upload-limit' in sys.argv:
index = sys.argv.index('--upload-limit')
sys.argv.pop(index)
sys.argv.pop(index)
if '--download-limit' in sys.argv:
index = sys.argv.index('--download-limit')
sys.argv.pop(index)
sys.argv.pop(index)
# locate trickle
trickle_executable = distspawn.find_executable('trickle')
if trickle_executable is None:
trickle_executable = distspawn.find_executable(
'trickle', path=":".join(sys.path))
if trickle_executable is None:
trickle_executable = distspawn.find_executable(
'trickle', path=":".join(os.environ.get('PATH')))
if trickle_executable:
LOG.info("Info: Starting trickle ...")
trickle_command = '{0} -d {1} -u {2} '.\
format(trickle_executable,
getattr(backup_args, 'download_limit') or -1,
getattr(backup_args, 'upload_limit') or -1)
backup_args.__dict__['trickle_command'] = trickle_command
if backup_args.config:
backup_args.__dict__['tmp_file'] = conf_file.name
# maintain env variable not to get into infinite loop
if "tricklecount" in os.environ:
tricklecount = int(os.environ.get("tricklecount", 1))
tricklecount += 1
os.environ["tricklecount"] = str(tricklecount)
else:
os.environ["tricklecount"] = str(1)
else:
LOG.warning("Trickle not found. Switching to normal mode without "
"limiting bandwidth")
if backup_args.config:
# remove index tmp_file from backup arguments dict
utils.delete_file(conf_file.name)
return backup_args
def freezer_main(backup_args):
"""Freezer main loop for job execution.
"""
if not backup_args.quiet:
LOG.info('log file at {0}'.format(CONF.get('log_file')))
if backup_args.max_priority:
utils.set_max_process_priority()
backup_args.__dict__['hostname_backup_name'] = "{0}_{1}".format(
backup_args.hostname, backup_args.backup_name)
validator.validate(backup_args)
work_dir = backup_args.work_dir
max_segment_size = backup_args.max_segment_size
if (backup_args.storage == 'swift'
or backup_args.backup_media in ['nova', 'cinder', 'cindernative']):
backup_args.client_manager = get_client_manager(backup_args.__dict__)
if backup_args.storages:
storage = multiple.MultipleStorage(work_dir, [
storage_from_dict(x, work_dir, max_segment_size)
for x in backup_args.storages
])
else:
storage = storage_from_dict(backup_args.__dict__, work_dir,
max_segment_size)
backup_args.engine = tar_engine.TarBackupEngine(
backup_args.compression,
backup_args.dereference_symlink, backup_args.exclude, storage,
winutils.is_windows(), backup_args.max_segment_size,
backup_args.encrypt_pass_file, backup_args.dry_run)
if hasattr(backup_args, 'trickle_command'):
if "tricklecount" in os.environ:
if int(os.environ.get("tricklecount")) > 1:
LOG.critical("[*] Trickle seems to be not working, Switching "
"to normal mode ")
return run_job(backup_args, storage)
freezer_command = '{0} {1}'.format(backup_args.trickle_command,
' '.join(sys.argv))
LOG.debug('Trickle command: {0}'.format(freezer_command))
process = subprocess.Popen(freezer_command.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=os.environ.copy())
while process.poll() is None:
line = process.stdout.readline().strip()
if line != '':
print(line)
output, error = process.communicate()
if hasattr(backup_args, 'tmp_file'):
utils.delete_file(backup_args.tmp_file)
if process.returncode:
LOG.warn("[*] Trickle Error: {0}".format(error))
LOG.info("[*] Switching to work without trickle ...")
return run_job(backup_args, storage)
else:
return run_job(backup_args, storage)
import threading
import time
from apscheduler.schedulers.background import BackgroundScheduler
from distutils import spawn
from oslo_config import cfg
from oslo_log import log
from freezer.apiclient import client
from freezer.scheduler import arguments
from freezer.scheduler import scheduler_job
from freezer.scheduler import shell
from freezer.scheduler import utils
from freezer.utils import winutils
if winutils.is_windows():
from win_daemon import Daemon
from win_daemon import NoDaemon
else:
from daemon import Daemon
from daemon import NoDaemon
CONF = cfg.CONF
LOG = log.getLogger(__name__)
class FreezerScheduler(object):
def __init__(self, apiclient, interval, job_path):
# config_manager
self.client = apiclient
self.freezerc_executable = spawn.find_executable('freezer-agent')
def freezer_main(backup_args):
"""Freezer main loop for job execution.
"""
if not backup_args.quiet:
LOG.info('log file at {0}'.format(CONF.get('log_file')))
if backup_args.max_priority:
utils.set_max_process_priority()
backup_args.__dict__['hostname_backup_name'] = "{0}_{1}".format(
backup_args.hostname, backup_args.backup_name)
validator.validate(backup_args)
work_dir = backup_args.work_dir
max_segment_size = backup_args.max_segment_size
if backup_args.storage == 'swift' or (
backup_args.backup_media in ['nova', 'cinder', 'cindernative']):
backup_args.client_manager = get_client_manager(backup_args.__dict__)
if backup_args.storages:
storage = multiple.MultipleStorage(
work_dir,
[storage_from_dict(x, work_dir, max_segment_size)
for x in backup_args.storages])
else:
storage = storage_from_dict(backup_args.__dict__, work_dir,
max_segment_size)
backup_args.engine = tar_engine.TarBackupEngine(
backup_args.compression,
backup_args.dereference_symlink,
backup_args.exclude,
storage,
winutils.is_windows(),
backup_args.max_segment_size,
backup_args.encrypt_pass_file,
backup_args.dry_run)
if hasattr(backup_args, 'trickle_command'):
if "tricklecount" in os.environ:
if int(os.environ.get("tricklecount")) > 1:
LOG.critical("[*] Trickle seems to be not working, Switching "
"to normal mode ")
return run_job(backup_args, storage)
freezer_command = '{0} {1}'.format(backup_args.trickle_command,
' '.join(sys.argv))
LOG.debug('Trickle command: {0}'.format(freezer_command))
process = subprocess.Popen(freezer_command.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=os.environ.copy())
while process.poll() is None:
line = process.stdout.readline().strip()
if line != '':
print (line)
output, error = process.communicate()
if hasattr(backup_args, 'tmp_file'):
utils.delete_file(backup_args.tmp_file)
if process.returncode:
LOG.warn("[*] Trickle Error: {0}".format(error))
LOG.info("[*] Switching to work without trickle ...")
return run_job(backup_args, storage)
else:
return run_job(backup_args, storage)
def main():
doers = _get_doers(shell)
doers.update(_get_doers(utils))
possible_actions = doers.keys() + ['start', 'stop', 'status', 'reload']
arguments.parse_args(possible_actions)
arguments.setup_logging()
if CONF.action is None:
CONF.print_help()
return 65 # os.EX_DATAERR
if CONF.action not in ['start', 'stop', 'status', 'reload']:
sys.stderr.write("Using freezer-scheduler as a command line client is "
"deprecated. Please use the freezer command line tool"
" from python-freezerclient.")
apiclient = None
insecure = False
if CONF.insecure:
insecure = True
if CONF.no_api is False:
try:
apiclient = client.Client(opts=CONF, insecure=insecure)
if CONF.client_id:
apiclient.client_id = CONF.client_id
except Exception as e:
LOG.error(e)
print(e)
sys.exit(1)
else:
if winutils.is_windows():
print("--no-api mode is not available on windows")
return 69 # os.EX_UNAVAILABLE
if CONF.action in doers:
try:
return doers[CONF.action](apiclient, CONF)
except Exception as e:
LOG.error(e)
print('ERROR {0}'.format(e))
return 70 # os.EX_SOFTWARE
freezer_scheduler = FreezerScheduler(apiclient=apiclient,
interval=int(CONF.interval),
job_path=CONF.jobs_dir)
if CONF.no_daemon:
print('Freezer Scheduler running in no-daemon mode')
LOG.debug('Freezer Scheduler running in no-daemon mode')
daemon = NoDaemon(daemonizable=freezer_scheduler)
else:
if winutils.is_windows():
daemon = Daemon(daemonizable=freezer_scheduler,
interval=int(CONF.interval),
job_path=CONF.jobs_dir,
insecure=CONF.insecure)
else:
daemon = Daemon(daemonizable=freezer_scheduler)
if CONF.action == 'start':
daemon.start()
elif CONF.action == 'stop':
daemon.stop()
elif CONF.action == 'reload':
daemon.reload()
elif CONF.action == 'status':
daemon.status()
# os.RETURN_CODES are only available to posix like systems, on windows
# we need to translate the code to an actual number which is the equivalent
return 0 # os.EX_OK
def test_is_windows(self):
fake_os = Os()
os.name = fake_os
assert winutils.is_windows() is False
"""
import os
import sys
from freezer import __version__ as FREEZER_VERSION
from freezer.apiclient import client as api_client
from freezer.utils import winutils
from oslo_config import cfg
from oslo_log import log
CONF = cfg.CONF
_LOG = log.getLogger(__name__)
if winutils.is_windows():
DEFAULT_FREEZER_SCHEDULER_CONF_D = r'C:\.freezer\scheduler\conf.d'
else:
DEFAULT_FREEZER_SCHEDULER_CONF_D = '/etc/freezer/scheduler/conf.d'
def get_common_opts():
scheduler_conf_d = os.environ.get('FREEZER_SCHEDULER_CONF_D',
DEFAULT_FREEZER_SCHEDULER_CONF_D)
if not os.path.exists(DEFAULT_FREEZER_SCHEDULER_CONF_D):
try:
os.makedirs(DEFAULT_FREEZER_SCHEDULER_CONF_D)
except OSError as err:
_LOG.error('OS error: {0}'.format(err))
except IOError:
_LOG.error('Cannot create the directory {0}'
def openssl_path():
from freezer.utils import winutils
if winutils.is_windows():
return 'openssl'
else:
return find_executable('openssl')