def freezer_main(backup_args): """Freezer main loop for job execution. """ def configure_log_file_using_defaults(): """ Configure log file for freezer """ dry_run_message = '' if backup_args.dry_run: dry_run_message = '[DRY_RUN] ' def configure_logging(file_name): expanded_file_name = os.path.expanduser(file_name) expanded_dir_name = os.path.dirname(expanded_file_name) utils.create_dir(expanded_dir_name, do_log=False) logging.basicConfig( filename=expanded_file_name, level=logging.INFO, format=('%(asctime)s %(name)s %(levelname)s {0}%(message)s'. format(dry_run_message))) return expanded_file_name if backup_args.log_file: return configure_logging(backup_args.log_file) for file_name in ['/var/log/freezer.log', '~/.freezer/freezer.log']: try: return configure_logging(file_name) except IOError: pass raise Exception("Unable to write to log file") def set_max_process_priority(): """ Set freezer in max priority on the os """ # children processes inherit niceness from father try: logging.warning( '[*] Setting freezer execution with high CPU and I/O priority') PID = os.getpid() # Set cpu priority os.nice(-19) # Set I/O Priority to Real Time class with level 0 subprocess.call([ u'{0}'.format(utils.find_executable("ionice")), u'-c', u'1', u'-n', u'0', u'-t', u'-p', u'{0}'.format(PID) ]) except Exception as priority_error: logging.warning('[*] Priority: {0}'.format(priority_error)) try: log_file_name = configure_log_file_using_defaults() except Exception as err: fail(1, err, quiet=backup_args.quiet, do_log=False) if not backup_args.quiet: logging.info('log file at {0}'.format(log_file_name)) if backup_args.max_priority: set_max_process_priority() monkeypatch_socket_bandwidth(backup_args) backup_args.__dict__['hostname_backup_name'] = "{0}_{1}".format( backup_args.hostname, backup_args.backup_name) validator.validate(backup_args) if backup_args.storage == "swift": options = utils.OpenstackOptions.create_from_env() identity_api_version = (backup_args.os_identity_api_version or options.identity_api_version) client_manager = ClientManager( options=options, insecure=backup_args.insecure, swift_auth_version=identity_api_version, dry_run=backup_args.dry_run) storage = swift.SwiftStorage( client_manager, backup_args.container, backup_args.work_dir, backup_args.max_segment_size) backup_args.__dict__['client_manager'] = client_manager elif backup_args.storage == "local": storage = local.LocalStorage(backup_args.container, backup_args.work_dir) elif backup_args.storage == "ssh": storage = ssh.SshStorage( backup_args.container, backup_args.work_dir, backup_args.ssh_key, backup_args.ssh_username, backup_args.ssh_host, backup_args.ssh_port) else: raise Exception("Not storage found for name " + backup_args.storage) backup_args.__dict__['engine'] = tar_engine.TarBackupEngine( backup_args.compression, backup_args.dereference_symlink, backup_args.exclude, storage, winutils.is_windows(), backup_args.encrypt_pass_file, backup_args.dry_run) if hasattr(backup_args, 'trickle_command'): if "tricklecount" in os.environ: if int(os.environ.get("tricklecount")) > 1: logging.critical("[*] Trickle seems to be not working," " Switching to normal mode ") run_job(backup_args, storage) freezer_command = '{0} {1}'.format(backup_args.trickle_command, ' '.join(sys.argv)) process = subprocess.Popen(freezer_command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ.copy()) while process.poll() is None: print process.stdout.readline().rstrip() output, error = process.communicate() if process.returncode: logging.error("[*] Trickle Error: {0}".format(error)) logging.critical("[*] Switching to work without trickle ...") run_job(backup_args, storage) else: run_job(backup_args, storage)
def freezer_main(args={}): """Freezer main loop for job execution. """ global backup_args, arg_parse def configure_log_file_using_defaults(): """ Configure log file for freezer """ dry_run_message = "" if backup_args.dry_run: dry_run_message = "[DRY_RUN] " def configure_logging(file_name): expanded_file_name = os.path.expanduser(file_name) expanded_dir_name = os.path.dirname(expanded_file_name) utils.create_dir(expanded_dir_name, do_log=False) logging.basicConfig( filename=expanded_file_name, level=logging.INFO, format=("%(asctime)s %(name)s %(levelname)s {0}%(message)s".format(dry_run_message)), ) return expanded_file_name if backup_args.log_file: return configure_logging(backup_args.log_file) for file_name in ["/var/log/freezer.log", "~/.freezer/freezer.log"]: try: return configure_logging(file_name) except IOError: pass raise Exception("Unable to write to log file") def set_max_process_priority(): """ Set freezer in max priority on the os """ # children processes inherit niceness from father try: logging.warning("[*] Setting freezer execution with high CPU and I/O priority") PID = os.getpid() # Set cpu priority os.nice(-19) # Set I/O Priority to Real Time class with level 0 subprocess.call( [u"{0}".format(backup_args.ionice), u"-c", u"1", u"-n", u"0", u"-t", u"-p", u"{0}".format(PID)] ) except Exception as priority_error: logging.warning("[*] Priority: {0}".format(priority_error)) # Alternative arguments provision useful to run Freezer without # command line e.g. functional testing if args: backup_args.__dict__.update(args) elif len(sys.argv) < 2: arg_parse.print_help() sys.exit(1) if backup_args.version: print "freezer version {0}".format(backup_args.__version__) sys.exit(1) try: log_file_name = configure_log_file_using_defaults() except Exception as err: fail(1, err, do_log=False) if not backup_args.quiet: logging.info("log file at {0}".format(log_file_name)) if backup_args.max_priority: set_max_process_priority() monkeypatch_socket_bandwidth(backup_args) backup_args.__dict__["hostname_backup_name"] = "{0}_{1}".format(backup_args.hostname, backup_args.backup_name) Validator.validate(backup_args) if backup_args.storage == "swift": options = utils.OpenstackOptions.create_from_env() Validator.validate_env(options) identity_api_version = backup_args.os_identity_api_version or options.identity_api_version client_manager = ClientManager( options=options, insecure=backup_args.insecure, swift_auth_version=identity_api_version, dry_run=backup_args.dry_run, ) storage = swift.SwiftStorage( client_manager, backup_args.container, backup_args.work_dir, backup_args.max_segment_size ) backup_args.__dict__["client_manager"] = client_manager elif backup_args.storage == "local": storage = local.LocalStorage(backup_args.container, backup_args.work_dir) elif backup_args.storage == "ssh": storage = ssh.SshStorage( backup_args.container, backup_args.work_dir, backup_args.ssh_key, backup_args.ssh_username, backup_args.ssh_host, backup_args.ssh_port, ) else: raise Exception("Not storage found for name " + backup_args.storage) backup_args.__dict__["storage"] = storage backup_args.__dict__["engine"] = tar_engine.TarBackupEngine( backup_args.tar_path, backup_args.compression, backup_args.dereference_symlink, backup_args.exclude, storage, winutils.is_windows(), backup_args.openssl_path, backup_args.encrypt_pass_file, backup_args.dry_run, ) freezer_job = job.create_job(backup_args) freezer_job.execute() if backup_args.metadata_out == "-": metadata = freezer_job.get_metadata() if metadata: sys.stdout.write(json.dumps(metadata)) return backup_args
def freezer_main(backup_args): """Freezer main loop for job execution. """ def set_max_process_priority(): """ Set freezer in max priority on the os """ # children processes inherit niceness from father try: LOG.warning( '[*] Setting freezer execution with high CPU and I/O priority') PID = os.getpid() # Set cpu priority os.nice(-19) # Set I/O Priority to Real Time class with level 0 subprocess.call([ u'{0}'.format(utils.find_executable("ionice")), u'-c', u'1', u'-n', u'0', u'-t', u'-p', u'{0}'.format(PID) ]) except Exception as priority_error: LOG.warning('[*] Priority: {0}'.format(priority_error)) if not backup_args.quiet: LOG.info('log file at {0}'.format(CONF.get('log_file'))) if backup_args.max_priority: set_max_process_priority() bandwidth.monkeypatch_socket_bandwidth(backup_args) backup_args.__dict__['hostname_backup_name'] = "{0}_{1}".format( backup_args.hostname, backup_args.backup_name) validator.validate(backup_args) work_dir = backup_args.work_dir os_identity = backup_args.os_identity_api_version max_segment_size = backup_args.max_segment_size if backup_args.storages: storage = multiple.MultipleStorage( work_dir, [storage_from_dict(x, work_dir, max_segment_size, os_identity) for x in backup_args.storages]) else: storage = storage_from_dict(backup_args.__dict__, work_dir, max_segment_size, os_identity) backup_args.__dict__['engine'] = tar_engine.TarBackupEngine( backup_args.compression, backup_args.dereference_symlink, backup_args.exclude, storage, winutils.is_windows(), backup_args.encrypt_pass_file, backup_args.dry_run) if hasattr(backup_args, 'trickle_command'): if "tricklecount" in os.environ: if int(os.environ.get("tricklecount")) > 1: LOG.critical("[*] Trickle seems to be not working, Switching " "to normal mode ") run_job(backup_args, storage) freezer_command = '{0} {1}'.format(backup_args.trickle_command, ' '.join(sys.argv)) process = subprocess.Popen(freezer_command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ.copy()) while process.poll() is None: print(process.stdout.readline().rstrip()) output, error = process.communicate() if process.returncode: LOG.error("[*] Trickle Error: {0}".format(error)) LOG.critical("[*] Switching to work without trickle ...") run_job(backup_args, storage) else: run_job(backup_args, storage)
def test_monkeypatch(self): monkeypatch_socket_bandwidth(100, 100)
def freezer_main(args={}): """Freezer main loop for job execution. """ global backup_args, arg_parse def configure_log_file_using_defaults(): """ Configure log file for freezer """ dry_run_message = '' if backup_args.dry_run: dry_run_message = '[DRY_RUN] ' def configure_logging(file_name): expanded_file_name = os.path.expanduser(file_name) expanded_dir_name = os.path.dirname(expanded_file_name) create_dir(expanded_dir_name, do_log=False) logging.basicConfig( filename=expanded_file_name, level=logging.INFO, format=('%(asctime)s %(name)s %(levelname)s {0}%(message)s'. format(dry_run_message))) return expanded_file_name if backup_args.log_file: return configure_logging(backup_args.log_file) for file_name in ['/var/log/freezer.log', '~/.freezer/freezer.log']: try: return configure_logging(file_name) except IOError: pass raise Exception("Unable to write to log file") def set_max_process_priority(): """ Set freezer in max priority on the os """ # children processes inherit niceness from father try: logging.warning( '[*] Setting freezer execution with high CPU and I/O priority') PID = os.getpid() # Set cpu priority os.nice(-19) # Set I/O Priority to Real Time class with level 0 subprocess.call([ u'{0}'.format(backup_args.ionice), u'-c', u'1', u'-n', u'0', u'-t', u'-p', u'{0}'.format(PID) ]) except Exception as priority_error: logging.warning('[*] Priority: {0}'.format(priority_error)) # Alternative arguments provision useful to run Freezer without # command line e.g. functional testing if args: backup_args.__dict__.update(args) elif len(sys.argv) < 2: arg_parse.print_help() sys.exit(1) if backup_args.version: print "freezer version {0}".format(backup_args.__version__) sys.exit(1) try: log_file_name = configure_log_file_using_defaults() except Exception as err: fail(1, err, do_log=False) if not backup_args.quiet: logging.info('log file at {0}'.format(log_file_name)) if backup_args.max_priority: set_max_process_priority() monkeypatch_socket_bandwidth(backup_args) backup_args.__dict__['hostname_backup_name'] = "{0}_{1}".format( backup_args.hostname, backup_args.backup_name) Validator.validate(backup_args) if backup_args.storage == "swift": options = utils.OpenstackOptions.create_from_env() Validator.validate_env(options) client_manager = ClientManager( options, backup_args.insecure, backup_args.os_auth_ver, backup_args.dry_run) backup_args.__dict__['storage'] = swift.SwiftStorage( client_manager, backup_args.container, backup_args.work_dir, backup_args.max_segment_size) elif backup_args.storage == "local": backup_args.__dict__['storage'] = \ local.LocalStorage(backup_args.container) elif backup_args.storage == "ssh": if not (backup_args.ssh_key and backup_args.ssh_username and backup_args.ssh_host): raise Exception("Please provide ssh_key, " "ssh_username and ssh_host") backup_args.__dict__['storage'] = \ ssh.SshStorage(backup_args.container, backup_args.work_dir, backup_args.ssh_key, backup_args.ssh_username, backup_args.ssh_host) else: raise Exception("Not storage found for name " + backup_args.storage) freezer_job = job.create_job(backup_args) freezer_job.execute() if backup_args.metadata_out == '-': metadata = freezer_job.get_metadata() if metadata: sys.stdout.write(json.dumps(metadata)) return backup_args