Ejemplo n.º 1
0
def freezer_main(backup_args):
    """Freezer main loop for job execution.
    """

    def configure_log_file_using_defaults():
        """ Configure log file for freezer """

        dry_run_message = ''
        if backup_args.dry_run:
            dry_run_message = '[DRY_RUN] '

        def configure_logging(file_name):
            expanded_file_name = os.path.expanduser(file_name)
            expanded_dir_name = os.path.dirname(expanded_file_name)
            utils.create_dir(expanded_dir_name, do_log=False)
            logging.basicConfig(
                filename=expanded_file_name,
                level=logging.INFO,
                format=('%(asctime)s %(name)s %(levelname)s {0}%(message)s'.
                        format(dry_run_message)))
            return expanded_file_name

        if backup_args.log_file:
            return configure_logging(backup_args.log_file)

        for file_name in ['/var/log/freezer.log', '~/.freezer/freezer.log']:
            try:
                return configure_logging(file_name)
            except IOError:
                pass

        raise Exception("Unable to write to log file")

    def set_max_process_priority():
        """ Set freezer in max priority on the os """
        # children processes inherit niceness from father
        try:
            logging.warning(
                '[*] Setting freezer execution with high CPU and I/O priority')
            PID = os.getpid()
            # Set cpu priority
            os.nice(-19)
            # Set I/O Priority to Real Time class with level 0
            subprocess.call([
                u'{0}'.format(utils.find_executable("ionice")),
                u'-c', u'1', u'-n', u'0', u'-t',
                u'-p', u'{0}'.format(PID)
            ])
        except Exception as priority_error:
            logging.warning('[*] Priority: {0}'.format(priority_error))

    try:
        log_file_name = configure_log_file_using_defaults()
    except Exception as err:
        fail(1, err, quiet=backup_args.quiet, do_log=False)

    if not backup_args.quiet:
        logging.info('log file at {0}'.format(log_file_name))

    if backup_args.max_priority:
        set_max_process_priority()

    monkeypatch_socket_bandwidth(backup_args)

    backup_args.__dict__['hostname_backup_name'] = "{0}_{1}".format(
        backup_args.hostname, backup_args.backup_name)

    validator.validate(backup_args)

    if backup_args.storage == "swift":
        options = utils.OpenstackOptions.create_from_env()
        identity_api_version = (backup_args.os_identity_api_version or
                                options.identity_api_version)
        client_manager = ClientManager(
            options=options,
            insecure=backup_args.insecure,
            swift_auth_version=identity_api_version,
            dry_run=backup_args.dry_run)

        storage = swift.SwiftStorage(
            client_manager, backup_args.container, backup_args.work_dir,
            backup_args.max_segment_size)
        backup_args.__dict__['client_manager'] = client_manager
    elif backup_args.storage == "local":
        storage = local.LocalStorage(backup_args.container,
                                     backup_args.work_dir)
    elif backup_args.storage == "ssh":
        storage = ssh.SshStorage(
            backup_args.container, backup_args.work_dir, backup_args.ssh_key,
            backup_args.ssh_username, backup_args.ssh_host,
            backup_args.ssh_port)
    else:
        raise Exception("Not storage found for name " + backup_args.storage)

    backup_args.__dict__['engine'] = tar_engine.TarBackupEngine(
        backup_args.compression,
        backup_args.dereference_symlink,
        backup_args.exclude,
        storage,
        winutils.is_windows(),
        backup_args.encrypt_pass_file,
        backup_args.dry_run)

    if hasattr(backup_args, 'trickle_command'):
        if "tricklecount" in os.environ:
            if int(os.environ.get("tricklecount")) > 1:
                logging.critical("[*] Trickle seems to be not working,"
                                 " Switching to normal mode ")
                run_job(backup_args, storage)

        freezer_command = '{0} {1}'.format(backup_args.trickle_command,
                                           ' '.join(sys.argv))
        process = subprocess.Popen(freezer_command.split(),
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   env=os.environ.copy())
        while process.poll() is None:
            print process.stdout.readline().rstrip()
        output, error = process.communicate()

        if process.returncode:
            logging.error("[*] Trickle Error: {0}".format(error))
            logging.critical("[*] Switching to work without trickle ...")
            run_job(backup_args, storage)

    else:
        run_job(backup_args, storage)
Ejemplo n.º 2
0
def freezer_main(args={}):
    """Freezer main loop for job execution.
    """
    global backup_args, arg_parse

    def configure_log_file_using_defaults():
        """ Configure log file for freezer """

        dry_run_message = ""
        if backup_args.dry_run:
            dry_run_message = "[DRY_RUN] "

        def configure_logging(file_name):
            expanded_file_name = os.path.expanduser(file_name)
            expanded_dir_name = os.path.dirname(expanded_file_name)
            utils.create_dir(expanded_dir_name, do_log=False)
            logging.basicConfig(
                filename=expanded_file_name,
                level=logging.INFO,
                format=("%(asctime)s %(name)s %(levelname)s {0}%(message)s".format(dry_run_message)),
            )
            return expanded_file_name

        if backup_args.log_file:
            return configure_logging(backup_args.log_file)

        for file_name in ["/var/log/freezer.log", "~/.freezer/freezer.log"]:
            try:
                return configure_logging(file_name)
            except IOError:
                pass

        raise Exception("Unable to write to log file")

    def set_max_process_priority():
        """ Set freezer in max priority on the os """
        # children processes inherit niceness from father
        try:
            logging.warning("[*] Setting freezer execution with high CPU and I/O priority")
            PID = os.getpid()
            # Set cpu priority
            os.nice(-19)
            # Set I/O Priority to Real Time class with level 0
            subprocess.call(
                [u"{0}".format(backup_args.ionice), u"-c", u"1", u"-n", u"0", u"-t", u"-p", u"{0}".format(PID)]
            )
        except Exception as priority_error:
            logging.warning("[*] Priority: {0}".format(priority_error))

    # Alternative arguments provision useful to run Freezer without
    # command line e.g. functional testing
    if args:
        backup_args.__dict__.update(args)
    elif len(sys.argv) < 2:
        arg_parse.print_help()
        sys.exit(1)

    if backup_args.version:
        print "freezer version {0}".format(backup_args.__version__)
        sys.exit(1)

    try:
        log_file_name = configure_log_file_using_defaults()
    except Exception as err:
        fail(1, err, do_log=False)

    if not backup_args.quiet:
        logging.info("log file at {0}".format(log_file_name))

    if backup_args.max_priority:
        set_max_process_priority()

    monkeypatch_socket_bandwidth(backup_args)

    backup_args.__dict__["hostname_backup_name"] = "{0}_{1}".format(backup_args.hostname, backup_args.backup_name)

    Validator.validate(backup_args)

    if backup_args.storage == "swift":
        options = utils.OpenstackOptions.create_from_env()
        Validator.validate_env(options)
        identity_api_version = backup_args.os_identity_api_version or options.identity_api_version
        client_manager = ClientManager(
            options=options,
            insecure=backup_args.insecure,
            swift_auth_version=identity_api_version,
            dry_run=backup_args.dry_run,
        )

        storage = swift.SwiftStorage(
            client_manager, backup_args.container, backup_args.work_dir, backup_args.max_segment_size
        )
        backup_args.__dict__["client_manager"] = client_manager
    elif backup_args.storage == "local":
        storage = local.LocalStorage(backup_args.container, backup_args.work_dir)
    elif backup_args.storage == "ssh":
        storage = ssh.SshStorage(
            backup_args.container,
            backup_args.work_dir,
            backup_args.ssh_key,
            backup_args.ssh_username,
            backup_args.ssh_host,
            backup_args.ssh_port,
        )
    else:
        raise Exception("Not storage found for name " + backup_args.storage)

    backup_args.__dict__["storage"] = storage
    backup_args.__dict__["engine"] = tar_engine.TarBackupEngine(
        backup_args.tar_path,
        backup_args.compression,
        backup_args.dereference_symlink,
        backup_args.exclude,
        storage,
        winutils.is_windows(),
        backup_args.openssl_path,
        backup_args.encrypt_pass_file,
        backup_args.dry_run,
    )

    freezer_job = job.create_job(backup_args)
    freezer_job.execute()

    if backup_args.metadata_out == "-":
        metadata = freezer_job.get_metadata()
        if metadata:
            sys.stdout.write(json.dumps(metadata))

    return backup_args
Ejemplo n.º 3
0
def freezer_main(backup_args):
    """Freezer main loop for job execution.
    """
    def set_max_process_priority():
        """ Set freezer in max priority on the os """
        # children processes inherit niceness from father
        try:
            LOG.warning(
                '[*] Setting freezer execution with high CPU and I/O priority')
            PID = os.getpid()
            # Set cpu priority
            os.nice(-19)
            # Set I/O Priority to Real Time class with level 0
            subprocess.call([
                u'{0}'.format(utils.find_executable("ionice")),
                u'-c', u'1', u'-n', u'0', u'-t',
                u'-p', u'{0}'.format(PID)
            ])
        except Exception as priority_error:
            LOG.warning('[*] Priority: {0}'.format(priority_error))

    if not backup_args.quiet:
        LOG.info('log file at {0}'.format(CONF.get('log_file')))

    if backup_args.max_priority:
        set_max_process_priority()

    bandwidth.monkeypatch_socket_bandwidth(backup_args)

    backup_args.__dict__['hostname_backup_name'] = "{0}_{1}".format(
        backup_args.hostname, backup_args.backup_name)

    validator.validate(backup_args)

    work_dir = backup_args.work_dir
    os_identity = backup_args.os_identity_api_version
    max_segment_size = backup_args.max_segment_size
    if backup_args.storages:
        storage = multiple.MultipleStorage(
            work_dir,
            [storage_from_dict(x, work_dir, max_segment_size, os_identity)
             for x in backup_args.storages])
    else:
        storage = storage_from_dict(backup_args.__dict__, work_dir,
                                    max_segment_size, os_identity)

    backup_args.__dict__['engine'] = tar_engine.TarBackupEngine(
        backup_args.compression,
        backup_args.dereference_symlink,
        backup_args.exclude,
        storage,
        winutils.is_windows(),
        backup_args.encrypt_pass_file,
        backup_args.dry_run)

    if hasattr(backup_args, 'trickle_command'):
        if "tricklecount" in os.environ:
            if int(os.environ.get("tricklecount")) > 1:
                LOG.critical("[*] Trickle seems to be not working,  Switching "
                             "to normal mode ")
                run_job(backup_args, storage)

        freezer_command = '{0} {1}'.format(backup_args.trickle_command,
                                           ' '.join(sys.argv))
        process = subprocess.Popen(freezer_command.split(),
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   env=os.environ.copy())
        while process.poll() is None:
            print(process.stdout.readline().rstrip())
        output, error = process.communicate()

        if process.returncode:
            LOG.error("[*] Trickle Error: {0}".format(error))
            LOG.critical("[*] Switching to work without trickle ...")
            run_job(backup_args, storage)

    else:
        run_job(backup_args, storage)
Ejemplo n.º 4
0
 def test_monkeypatch(self):
     monkeypatch_socket_bandwidth(100, 100)
Ejemplo n.º 5
0
def freezer_main(args={}):
    """Freezer main loop for job execution.
    """
    global backup_args, arg_parse

    def configure_log_file_using_defaults():
        """ Configure log file for freezer """

        dry_run_message = ''
        if backup_args.dry_run:
            dry_run_message = '[DRY_RUN] '

        def configure_logging(file_name):
            expanded_file_name = os.path.expanduser(file_name)
            expanded_dir_name = os.path.dirname(expanded_file_name)
            create_dir(expanded_dir_name, do_log=False)
            logging.basicConfig(
                filename=expanded_file_name,
                level=logging.INFO,
                format=('%(asctime)s %(name)s %(levelname)s {0}%(message)s'.
                        format(dry_run_message)))
            return expanded_file_name

        if backup_args.log_file:
            return configure_logging(backup_args.log_file)

        for file_name in ['/var/log/freezer.log', '~/.freezer/freezer.log']:
            try:
                return configure_logging(file_name)
            except IOError:
                pass

        raise Exception("Unable to write to log file")

    def set_max_process_priority():
        """ Set freezer in max priority on the os """
        # children processes inherit niceness from father
        try:
            logging.warning(
                '[*] Setting freezer execution with high CPU and I/O priority')
            PID = os.getpid()
            # Set cpu priority
            os.nice(-19)
            # Set I/O Priority to Real Time class with level 0
            subprocess.call([
                u'{0}'.format(backup_args.ionice),
                u'-c', u'1', u'-n', u'0', u'-t',
                u'-p', u'{0}'.format(PID)
                ])
        except Exception as priority_error:
            logging.warning('[*] Priority: {0}'.format(priority_error))

    # Alternative arguments provision useful to run Freezer without
    # command line e.g. functional testing
    if args:
        backup_args.__dict__.update(args)
    elif len(sys.argv) < 2:
        arg_parse.print_help()
        sys.exit(1)

    if backup_args.version:
        print "freezer version {0}".format(backup_args.__version__)
        sys.exit(1)

    try:
        log_file_name = configure_log_file_using_defaults()
    except Exception as err:
        fail(1, err, do_log=False)

    if not backup_args.quiet:
        logging.info('log file at {0}'.format(log_file_name))

    if backup_args.max_priority:
        set_max_process_priority()

    monkeypatch_socket_bandwidth(backup_args)

    backup_args.__dict__['hostname_backup_name'] = "{0}_{1}".format(
        backup_args.hostname, backup_args.backup_name)

    Validator.validate(backup_args)

    if backup_args.storage == "swift":
        options = utils.OpenstackOptions.create_from_env()
        Validator.validate_env(options)
        client_manager = ClientManager(
            options,
            backup_args.insecure,
            backup_args.os_auth_ver,
            backup_args.dry_run)

        backup_args.__dict__['storage'] = swift.SwiftStorage(
            client_manager,
            backup_args.container,
            backup_args.work_dir,
            backup_args.max_segment_size)
    elif backup_args.storage == "local":
        backup_args.__dict__['storage'] = \
            local.LocalStorage(backup_args.container)
    elif backup_args.storage == "ssh":
        if not (backup_args.ssh_key and backup_args.ssh_username
                and backup_args.ssh_host):
            raise Exception("Please provide ssh_key, "
                            "ssh_username and ssh_host")
        backup_args.__dict__['storage'] = \
            ssh.SshStorage(backup_args.container,
                           backup_args.work_dir,
                           backup_args.ssh_key,
                           backup_args.ssh_username,
                           backup_args.ssh_host)
    else:
        raise Exception("Not storage found for name " + backup_args.storage)

    freezer_job = job.create_job(backup_args)
    freezer_job.execute()

    if backup_args.metadata_out == '-':
        metadata = freezer_job.get_metadata()
        if metadata:
            sys.stdout.write(json.dumps(metadata))

    return backup_args