def freezer_main(backup_args):
"""Freezer main loop for job execution.
"""
def configure_log_file_using_defaults():
""" Configure log file for freezer """
dry_run_message = ''
if backup_args.dry_run:
dry_run_message = '[DRY_RUN] '
def configure_logging(file_name):
expanded_file_name = os.path.expanduser(file_name)
expanded_dir_name = os.path.dirname(expanded_file_name)
utils.create_dir(expanded_dir_name, do_log=False)
logging.basicConfig(
filename=expanded_file_name,
level=logging.INFO,
format=('%(asctime)s %(name)s %(levelname)s {0}%(message)s'.
format(dry_run_message)))
return expanded_file_name
if backup_args.log_file:
return configure_logging(backup_args.log_file)
for file_name in ['/var/log/freezer.log', '~/.freezer/freezer.log']:
try:
return configure_logging(file_name)
except IOError:
pass
raise Exception("Unable to write to log file")
def set_max_process_priority():
""" Set freezer in max priority on the os """
# children processes inherit niceness from father
try:
logging.warning(
'[*] Setting freezer execution with high CPU and I/O priority')
PID = os.getpid()
# Set cpu priority
os.nice(-19)
# Set I/O Priority to Real Time class with level 0
subprocess.call([
u'{0}'.format(utils.find_executable("ionice")),
u'-c', u'1', u'-n', u'0', u'-t',
u'-p', u'{0}'.format(PID)
])
except Exception as priority_error:
logging.warning('[*] Priority: {0}'.format(priority_error))
try:
log_file_name = configure_log_file_using_defaults()
except Exception as err:
fail(1, err, quiet=backup_args.quiet, do_log=False)
if not backup_args.quiet:
logging.info('log file at {0}'.format(log_file_name))
if backup_args.max_priority:
set_max_process_priority()
monkeypatch_socket_bandwidth(backup_args)
backup_args.__dict__['hostname_backup_name'] = "{0}_{1}".format(
backup_args.hostname, backup_args.backup_name)
validator.validate(backup_args)
if backup_args.storage == "swift":
options = utils.OpenstackOptions.create_from_env()
identity_api_version = (backup_args.os_identity_api_version or
options.identity_api_version)
client_manager = ClientManager(
options=options,
insecure=backup_args.insecure,
swift_auth_version=identity_api_version,
dry_run=backup_args.dry_run)
storage = swift.SwiftStorage(
client_manager, backup_args.container, backup_args.work_dir,
backup_args.max_segment_size)
backup_args.__dict__['client_manager'] = client_manager
elif backup_args.storage == "local":
storage = local.LocalStorage(backup_args.container,
backup_args.work_dir)
elif backup_args.storage == "ssh":
storage = ssh.SshStorage(
backup_args.container, backup_args.work_dir, backup_args.ssh_key,
backup_args.ssh_username, backup_args.ssh_host,
backup_args.ssh_port)
else:
raise Exception("Not storage found for name " + backup_args.storage)
backup_args.__dict__['engine'] = tar_engine.TarBackupEngine(
backup_args.compression,
backup_args.dereference_symlink,
backup_args.exclude,
storage,
winutils.is_windows(),
backup_args.encrypt_pass_file,
backup_args.dry_run)
if hasattr(backup_args, 'trickle_command'):
if "tricklecount" in os.environ:
if int(os.environ.get("tricklecount")) > 1:
logging.critical("[*] Trickle seems to be not working,"
" Switching to normal mode ")
run_job(backup_args, storage)
freezer_command = '{0} {1}'.format(backup_args.trickle_command,
' '.join(sys.argv))
process = subprocess.Popen(freezer_command.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=os.environ.copy())
while process.poll() is None:
print process.stdout.readline().rstrip()
output, error = process.communicate()
if process.returncode:
logging.error("[*] Trickle Error: {0}".format(error))
logging.critical("[*] Switching to work without trickle ...")
run_job(backup_args, storage)
else:
run_job(backup_args, storage)
def freezer_main(args={}):
"""Freezer main loop for job execution.
"""
global backup_args, arg_parse
def configure_log_file_using_defaults():
""" Configure log file for freezer """
dry_run_message = ""
if backup_args.dry_run:
dry_run_message = "[DRY_RUN] "
def configure_logging(file_name):
expanded_file_name = os.path.expanduser(file_name)
expanded_dir_name = os.path.dirname(expanded_file_name)
utils.create_dir(expanded_dir_name, do_log=False)
logging.basicConfig(
filename=expanded_file_name,
level=logging.INFO,
format=("%(asctime)s %(name)s %(levelname)s {0}%(message)s".format(dry_run_message)),
)
return expanded_file_name
if backup_args.log_file:
return configure_logging(backup_args.log_file)
for file_name in ["/var/log/freezer.log", "~/.freezer/freezer.log"]:
try:
return configure_logging(file_name)
except IOError:
pass
raise Exception("Unable to write to log file")
def set_max_process_priority():
""" Set freezer in max priority on the os """
# children processes inherit niceness from father
try:
logging.warning("[*] Setting freezer execution with high CPU and I/O priority")
PID = os.getpid()
# Set cpu priority
os.nice(-19)
# Set I/O Priority to Real Time class with level 0
subprocess.call(
[u"{0}".format(backup_args.ionice), u"-c", u"1", u"-n", u"0", u"-t", u"-p", u"{0}".format(PID)]
)
except Exception as priority_error:
logging.warning("[*] Priority: {0}".format(priority_error))
# Alternative arguments provision useful to run Freezer without
# command line e.g. functional testing
if args:
backup_args.__dict__.update(args)
elif len(sys.argv) < 2:
arg_parse.print_help()
sys.exit(1)
if backup_args.version:
print "freezer version {0}".format(backup_args.__version__)
sys.exit(1)
try:
log_file_name = configure_log_file_using_defaults()
except Exception as err:
fail(1, err, do_log=False)
if not backup_args.quiet:
logging.info("log file at {0}".format(log_file_name))
if backup_args.max_priority:
set_max_process_priority()
monkeypatch_socket_bandwidth(backup_args)
backup_args.__dict__["hostname_backup_name"] = "{0}_{1}".format(backup_args.hostname, backup_args.backup_name)
Validator.validate(backup_args)
if backup_args.storage == "swift":
options = utils.OpenstackOptions.create_from_env()
Validator.validate_env(options)
identity_api_version = backup_args.os_identity_api_version or options.identity_api_version
client_manager = ClientManager(
options=options,
insecure=backup_args.insecure,
swift_auth_version=identity_api_version,
dry_run=backup_args.dry_run,
)
storage = swift.SwiftStorage(
client_manager, backup_args.container, backup_args.work_dir, backup_args.max_segment_size
)
backup_args.__dict__["client_manager"] = client_manager
elif backup_args.storage == "local":
storage = local.LocalStorage(backup_args.container, backup_args.work_dir)
elif backup_args.storage == "ssh":
storage = ssh.SshStorage(
backup_args.container,
backup_args.work_dir,
backup_args.ssh_key,
backup_args.ssh_username,
backup_args.ssh_host,
backup_args.ssh_port,
)
else:
raise Exception("Not storage found for name " + backup_args.storage)
backup_args.__dict__["storage"] = storage
backup_args.__dict__["engine"] = tar_engine.TarBackupEngine(
backup_args.tar_path,
backup_args.compression,
backup_args.dereference_symlink,
backup_args.exclude,
storage,
winutils.is_windows(),
backup_args.openssl_path,
backup_args.encrypt_pass_file,
backup_args.dry_run,
)
freezer_job = job.create_job(backup_args)
freezer_job.execute()
if backup_args.metadata_out == "-":
metadata = freezer_job.get_metadata()
if metadata:
sys.stdout.write(json.dumps(metadata))
return backup_args
def freezer_main(backup_args):
"""Freezer main loop for job execution.
"""
def set_max_process_priority():
""" Set freezer in max priority on the os """
# children processes inherit niceness from father
try:
LOG.warning(
'[*] Setting freezer execution with high CPU and I/O priority')
PID = os.getpid()
# Set cpu priority
os.nice(-19)
# Set I/O Priority to Real Time class with level 0
subprocess.call([
u'{0}'.format(utils.find_executable("ionice")),
u'-c', u'1', u'-n', u'0', u'-t',
u'-p', u'{0}'.format(PID)
])
except Exception as priority_error:
LOG.warning('[*] Priority: {0}'.format(priority_error))
if not backup_args.quiet:
LOG.info('log file at {0}'.format(CONF.get('log_file')))
if backup_args.max_priority:
set_max_process_priority()
bandwidth.monkeypatch_socket_bandwidth(backup_args)
backup_args.__dict__['hostname_backup_name'] = "{0}_{1}".format(
backup_args.hostname, backup_args.backup_name)
validator.validate(backup_args)
work_dir = backup_args.work_dir
os_identity = backup_args.os_identity_api_version
max_segment_size = backup_args.max_segment_size
if backup_args.storages:
storage = multiple.MultipleStorage(
work_dir,
[storage_from_dict(x, work_dir, max_segment_size, os_identity)
for x in backup_args.storages])
else:
storage = storage_from_dict(backup_args.__dict__, work_dir,
max_segment_size, os_identity)
backup_args.__dict__['engine'] = tar_engine.TarBackupEngine(
backup_args.compression,
backup_args.dereference_symlink,
backup_args.exclude,
storage,
winutils.is_windows(),
backup_args.encrypt_pass_file,
backup_args.dry_run)
if hasattr(backup_args, 'trickle_command'):
if "tricklecount" in os.environ:
if int(os.environ.get("tricklecount")) > 1:
LOG.critical("[*] Trickle seems to be not working, Switching "
"to normal mode ")
run_job(backup_args, storage)
freezer_command = '{0} {1}'.format(backup_args.trickle_command,
' '.join(sys.argv))
process = subprocess.Popen(freezer_command.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=os.environ.copy())
while process.poll() is None:
print(process.stdout.readline().rstrip())
output, error = process.communicate()
if process.returncode:
LOG.error("[*] Trickle Error: {0}".format(error))
LOG.critical("[*] Switching to work without trickle ...")
run_job(backup_args, storage)
else:
run_job(backup_args, storage)
def test_monkeypatch(self):
monkeypatch_socket_bandwidth(100, 100)
def freezer_main(args={}):
"""Freezer main loop for job execution.
"""
global backup_args, arg_parse
def configure_log_file_using_defaults():
""" Configure log file for freezer """
dry_run_message = ''
if backup_args.dry_run:
dry_run_message = '[DRY_RUN] '
def configure_logging(file_name):
expanded_file_name = os.path.expanduser(file_name)
expanded_dir_name = os.path.dirname(expanded_file_name)
create_dir(expanded_dir_name, do_log=False)
logging.basicConfig(
filename=expanded_file_name,
level=logging.INFO,
format=('%(asctime)s %(name)s %(levelname)s {0}%(message)s'.
format(dry_run_message)))
return expanded_file_name
if backup_args.log_file:
return configure_logging(backup_args.log_file)
for file_name in ['/var/log/freezer.log', '~/.freezer/freezer.log']:
try:
return configure_logging(file_name)
except IOError:
pass
raise Exception("Unable to write to log file")
def set_max_process_priority():
""" Set freezer in max priority on the os """
# children processes inherit niceness from father
try:
logging.warning(
'[*] Setting freezer execution with high CPU and I/O priority')
PID = os.getpid()
# Set cpu priority
os.nice(-19)
# Set I/O Priority to Real Time class with level 0
subprocess.call([
u'{0}'.format(backup_args.ionice),
u'-c', u'1', u'-n', u'0', u'-t',
u'-p', u'{0}'.format(PID)
])
except Exception as priority_error:
logging.warning('[*] Priority: {0}'.format(priority_error))
# Alternative arguments provision useful to run Freezer without
# command line e.g. functional testing
if args:
backup_args.__dict__.update(args)
elif len(sys.argv) < 2:
arg_parse.print_help()
sys.exit(1)
if backup_args.version:
print "freezer version {0}".format(backup_args.__version__)
sys.exit(1)
try:
log_file_name = configure_log_file_using_defaults()
except Exception as err:
fail(1, err, do_log=False)
if not backup_args.quiet:
logging.info('log file at {0}'.format(log_file_name))
if backup_args.max_priority:
set_max_process_priority()
monkeypatch_socket_bandwidth(backup_args)
backup_args.__dict__['hostname_backup_name'] = "{0}_{1}".format(
backup_args.hostname, backup_args.backup_name)
Validator.validate(backup_args)
if backup_args.storage == "swift":
options = utils.OpenstackOptions.create_from_env()
Validator.validate_env(options)
client_manager = ClientManager(
options,
backup_args.insecure,
backup_args.os_auth_ver,
backup_args.dry_run)
backup_args.__dict__['storage'] = swift.SwiftStorage(
client_manager,
backup_args.container,
backup_args.work_dir,
backup_args.max_segment_size)
elif backup_args.storage == "local":
backup_args.__dict__['storage'] = \
local.LocalStorage(backup_args.container)
elif backup_args.storage == "ssh":
if not (backup_args.ssh_key and backup_args.ssh_username
and backup_args.ssh_host):
raise Exception("Please provide ssh_key, "
"ssh_username and ssh_host")
backup_args.__dict__['storage'] = \
ssh.SshStorage(backup_args.container,
backup_args.work_dir,
backup_args.ssh_key,
backup_args.ssh_username,
backup_args.ssh_host)
else:
raise Exception("Not storage found for name " + backup_args.storage)
freezer_job = job.create_job(backup_args)
freezer_job.execute()
if backup_args.metadata_out == '-':
metadata = freezer_job.get_metadata()
if metadata:
sys.stdout.write(json.dumps(metadata))
return backup_args