Exemplo n.º 1
0
def run(args, user, log_path):
    """Run the WMF auto reimage according to command line arguments.

    Arguments:
    args     -- parsed command line arguments
    user     -- the user that launched the script, for auditing purposes
    log_path -- the path of the logfile
    """
    previous = None  # Previous state in conftool
    rename_from = None  # In case of host rename, hold the previous hostname

    # Validate hosts have a signed Puppet certificate
    if not args.new:
        lib.validate_hosts([args.host], no_raise=args.no_verify)

    # Set Icinga downtime
    if not args.new and not args.no_downtime:
        lib.icinga_downtime(args.host, user, args.phab_task_id)

    # Depool via conftool
    if args.conftool and not args.new:
        previous = lib.conftool_depool(args.host, pooled=args.conftool_value)
        lib.print_line('Waiting 3 minutes to let the host drain',
                       host=args.host)
        time.sleep(180)

    if args.no_pxe:
        lib.print_line('Skipping PXE reboot', host=args.host)
        if (not lib.validate_hosts([args.host], no_raise=True)
                and lib.puppet_check_cert_to_sign(args.host) == 1):
            # There is no signed or pending signing certificate for the host
            lib.puppet_generate_cert(args.host)
    else:
        lib.puppet_remove_host(args.host)  # Cleanup Puppet

        # Reboot into PXE mode to start the reimage
        lib.set_pxe_boot(args.host, args.mgmt)
        status = lib.ipmitool_command(args.mgmt,
                                      ['chassis', 'power', 'status'])
        if status.startswith('Chassis Power is off'):
            lib.print_line('Current power status is off, powering on',
                           host=args.host)
            ipmi_command = ['chassis', 'power', 'on']
        else:
            lib.print_line('Power cycling', host=args.host)
            ipmi_command = ['chassis', 'power', 'cycle']

        lib.print_line(lib.ipmitool_command(args.mgmt,
                                            ipmi_command).rstrip('\n'),
                       host=args.host)

        # If the host is renamed, swap the hostnames now
        if args.rename is not None:
            rename_from = args.host
            args.host = args.rename
            args.mgmt = args.rename_mgmt

        # Wait that the host is booting into the installer using Cumin's direct backend
        lib.wait_reboot(args.host,
                        start=datetime.utcnow(),
                        installer_key=True,
                        debian_installer=True)
        # Wait for the reboot into the new system
        time.sleep(
            30
        )  # Avoid race conditions, the host is in the d-i, need to wait anyway
        lib.wait_reboot(args.host, start=datetime.utcnow(), installer_key=True)

        # Generate the Puppet certificate and signing request
        if lib.detect_init_system(args.host) == 'systemd':
            lib.puppet_generate_cert(args.host)

    # Sign the new Puppet certificate
    if lib.puppet_wait_cert_and_sign(args.host):
        if args.mask:  # Mask systemd services
            lib.mask_systemd_services(args.host, args.mask)

        lib.puppet_first_run(args.host)
        # Ensure the host is in Icinga
        lib.run_puppet([lib.resolve_dns(lib.ICINGA_DOMAIN, 'CNAME')],
                       no_raise=True)
        lib.icinga_downtime(args.host, user, args.phab_task_id)

    lib.check_bios_bootparams(args.host, args.mgmt)

    # Issue a reboot and wait for it and also for Puppet to complete
    if not args.no_reboot:
        reboot_time = datetime.utcnow()
        # Ensure the host is in the known hosts
        lib.run_puppet([socket.getfqdn()], no_raise=True)
        lib.reboot_host(args.host)
        boot_time = datetime.utcnow()
        lib.wait_reboot(args.host, start=reboot_time)
        lib.wait_puppet_run(args.host, start=boot_time)

    # Run Apache fast test
    if args.apache:
        lib.run_apache_fast_test(args.host)

    # The unmask is *not* done automatically, the commands to unmask are printed and logged
    if args.mask:
        lib.print_unmask_message(args.host, args.mask)

    # The repool is *not* done automatically, the command to repool is printed and logged
    if args.conftool:
        lib.print_repool_message(previous,
                                 rename_from=rename_from,
                                 rename_to=args.rename)

    lib.print_line('Reimage completed', host=args.host)
Exemplo n.º 2
0
def run(args, user, log_path):
    """Run the WMF auto reimage according to command line arguments.

    Arguments:
    args     -- parsed command line arguments
    user     -- the user that launched the script, for auditing purposes
    log_path -- the path of the logfile
    """
    previous = None  # Previous state in conftool
    rename_from = None  # In case of host rename, hold the previous hostname
    cert_fingerprint = None

    # Validate hosts have a signed Puppet certificate
    if not args.new:
        lib.validate_hosts([args.host], no_raise=args.no_verify)

    # Set Icinga downtime
    if not args.new and not args.no_downtime:
        lib.icinga_downtime(args.host, user, args.phab_task_id)

    # Depool via conftool
    if args.conftool and not args.new:
        previous = lib.conftool_depool(args.host, pooled=args.conftool_value)
        lib.print_line('Waiting 3 minutes to let the host drain',
                       host=args.host)
        time.sleep(180)

    if args.no_pxe:
        lib.print_line('Skipping PXE reboot', host=args.host)
        if not lib.validate_hosts([args.host], no_raise=True):
            # There is no valid cert for the host, remove local cert and re-generate it
            lib.puppet_remove_local_cert(args.host, installer=True)
            ret = lib.puppet_check_cert_to_sign(args.host, lib.CERT_DESTROY)
            if ret != 1:
                raise RuntimeError((
                    'There was an error checking the certificate. puppet_check_cert_to_sign() '
                    'should have returned 1, got {ret} instead').format(
                        ret=ret))

            cert_fingerprint = lib.puppet_generate_cert(args.host)
    else:
        lib.puppet_remove_host(args.host)  # Cleanup Puppet
        lib.debmonitor_remove_host(args.host)

        # Reboot into PXE mode to start the reimage
        lib.set_pxe_boot(args.host, args.mgmt)
        status = lib.ipmitool_command(args.mgmt,
                                      ['chassis', 'power', 'status'])
        if status.startswith('Chassis Power is off'):
            lib.print_line('Current power status is off, powering on',
                           host=args.host)
            ipmi_command = ['chassis', 'power', 'on']
        else:
            lib.print_line('Power cycling', host=args.host)
            ipmi_command = ['chassis', 'power', 'cycle']

        lib.print_line(lib.ipmitool_command(args.mgmt,
                                            ipmi_command).rstrip('\n'),
                       host=args.host)

        # If the host is renamed, swap the hostnames now
        if args.rename is not None:
            rename_from = args.host
            args.host = args.rename
            args.mgmt = args.rename_mgmt

        # Wait that the host is booting into the installer using Cumin's direct backend
        lib.wait_reboot(args.host,
                        start=datetime.utcnow(),
                        installer_key=True,
                        debian_installer=True)
        # Wait for the reboot into the new system
        time.sleep(
            30
        )  # Avoid race conditions, the host is in the d-i, need to wait anyway
        lib.wait_reboot(args.host, start=datetime.utcnow(), installer_key=True)

        # Generate the Puppet certificate and signing request
        cert_fingerprint = lib.puppet_generate_cert(args.host)

    # Sign the new Puppet certificate
    if lib.puppet_wait_cert_and_sign(args.host, cert_fingerprint):
        if args.mask:  # Mask systemd services
            lib.mask_systemd_services(args.host, args.mask)

        # Downtime the host on Icinga with delay to give time to compile the Puppet catalog
        # and export its resources
        downtime_debug = ''
        if args.debug:
            downtime_debug = '-v '
        downtime_command = (
            '/bin/sleep 120 && /usr/bin/cookbook {verbose}sre.hosts.downtime '
            '--force-puppet -H 2 -r REIMAGE {host}').format(
                verbose=downtime_debug, host=args.host)

        downtime = subprocess.Popen(downtime_command, shell=True)
        lib.print_line('Scheduled delayed downtime on Icinga', host=args.host)

        lib.puppet_first_run(args.host)

        try:
            downtime.wait(timeout=180)
            downtime_success = (downtime.returncode == 0)
            downtime_message = 'returned {ret}'.format(ret=downtime.returncode)
        except subprocess.TimeoutExpired:
            downtime_success = False
            downtime_message = 'timed out'

        if not downtime_success:
            lib.print_line((
                'WARNING: failed to downtime host on Icinga, wmf-downtime-host '
                '{msg}').format(msg=downtime_message))

    lib.check_bios_bootparams(args.host, args.mgmt)

    # Issue a reboot and wait for it and also for Puppet to complete
    if not args.no_reboot:
        reboot_time = datetime.utcnow()
        # Ensure the host is in the known hosts
        lib.run_puppet([socket.getfqdn()], no_raise=True)
        lib.reboot_host(args.host)
        boot_time = datetime.utcnow()
        lib.wait_reboot(args.host, start=reboot_time)
        lib.wait_puppet_run(args.host, start=boot_time)

    # Run httpbb
    if args.apache:
        lib.run_httpbb(args.host)

    # The unmask is *not* done automatically, the commands to unmask are printed and logged
    if args.mask:
        lib.print_unmask_message(args.host, args.mask)

    # The repool is *not* done automatically, the command to repool is printed and logged
    if args.conftool:
        lib.print_repool_message(previous,
                                 rename_from=rename_from,
                                 rename_to=args.rename)

    lib.print_line('Reimage completed', host=args.host)