def wbatch_begin_node(node_name):
    wbatch_new_file("create_{}_node.bat".format(node_name))
    wbatch_file_header("{} VM".format(node_name))
    wbatch_find_vbm()
    wbatch_mkdirs()
    basedisk = "{}.vdi".format(conf.get_base_disk_name())

    replace = {"BASEDISK": basedisk}
    wbatch_write_template("template-begin_node_bat", replace)
def wbatch_begin_node(node_name):
    wbatch_new_file("create_{}_node.bat".format(node_name))
    wbatch_file_header("{} VM".format(node_name))
    wbatch_find_vbm()
    wbatch_mkdirs()
    basedisk = "{}.vdi".format(conf.get_base_disk_name())

    replace = {"BASEDISK": basedisk}
    wbatch_write_template("template-begin_node_bat", replace)
Example #3
0
def base_disk_delete():
    base_disk_name = conf.get_base_disk_name()

    base_disk_path = vm.get_disk_path(base_disk_name)
    output = vm.virsh("list", "--uuid", "--all")
    for uuid in output.splitlines():
        if uuid == "":
            continue
        logger.debug("Candidate for deletion: %s", uuid)
        domstats = vm.virsh("domstats", uuid, "--block", "--backing")
        if re.search("block.*path.*{}".format(base_disk_path), domstats):
            logger.info("Deleting VM %s", uuid)
            vm.vm_delete(uuid)
    vm.disk_delete(base_disk_name)
Example #4
0
def get_base_disk_path():
    return os.path.join(conf.img_dir, conf.get_base_disk_name() + ".vdi")
Example #5
0
def vm_install_base():
    vm_name = "base"

    conf.vm[vm_name] = conf.VMconfig(vm_name)

    base_disk_name = conf.get_base_disk_name()

    vm.vm_delete(vm_name)

    if base_disk_exists():
        logger.info("Deleting existing basedisk.")
        base_disk_delete()

    vm_config = conf.vm[vm_name]

    if conf.do_build:
        install_iso = iso_image.find_install_iso()
    else:
        install_iso = os.path.join(conf.img_dir, conf.iso_image.name)

    logger.info("Install ISO:\n\t%s", install_iso)

    autostart.autostart_reset()

    autostart.autostart_queue("osbash/base_fixups.sh")

    autostart.autostart_from_config(conf.base_install_scripts)

    autostart.autostart_queue("zero_empty.sh", "shutdown.sh")

    base_disk_size = 10000
    vm.disk_create(base_disk_name, base_disk_size)

    libvirt_connect_uri = "qemu:///system"
    virt_install_call = [
        "sudo", "virt-install", "--connect={}".format(libvirt_connect_uri)
    ]
    call_args = virt_install_call
    call_args.extend(["--name", vm_name])
    call_args.extend(["--ram", str(vm_config.vm_mem)])
    call_args.extend(["--vcpus", str(1)])
    call_args.extend(["--os-type", "linux"])
    call_args.extend(["--cdrom", install_iso])

    call_args.extend([
        "--disk", "vol={}/{},cache=none".format(vm.kvm_vol_pool,
                                                base_disk_name)
    ])

    if conf.vm_ui == "headless":
        call_args.extend(("--graphics", "none", "--noautoconsole"))
    elif conf.vm_ui == "vnc":
        call_args.extend(("--graphics", "vnc,listen=127.0.0.1"))
    # Default (no extra argument) is gui option: should open a console viewer
    call_args.append("--wait=-1")

    import subprocess
    errout = subprocess.STDOUT
    logger.debug("virt-install call: %s", ' '.join(call_args))
    logger.debug("virt-install call: %s", call_args)
    vm.virsh_log(call_args)
    subprocess.Popen(call_args, stderr=errout)

    while True:
        if vm.vm_is_running(vm_name):
            break
        print('.', end='')
        time.sleep(1)

    delay = 5
    logger.info("\nWaiting %d seconds for VM %s to come up.", delay, vm_name)
    cs.conditional_sleep(delay)

    logger.info("Booting into distribution installer.")
    distro_boot.distro_start_installer(vm_config)

    # Prevent "time stamp from the future" due to race between two sudos for
    # virt-install (background) above and virsh below
    time.sleep(1)

    logger.info("Waiting for VM %s to be defined.", vm_name)
    while True:
        if vm.vm_is_running(vm_name):
            break
        time.sleep(1)
        print(".")

    ssh_ip = vm.node_to_ip(vm_name)
    conf.vm[vm_name].ssh_ip = ssh_ip

    logger.info("Waiting for ping returning from %s.", ssh_ip)
    hf.wait_for_ping(ssh_ip)

    autostart.autostart_and_wait(vm_name)

    vm.vm_wait_for_shutdown(vm_name)

    logger.info("Compacting %s.", base_disk_name)
    vm.disk_compress(base_disk_name)

    vm.virsh("undefine", vm_name)

    del conf.vm[vm_name]

    logger.info("Base disk created.")

    logger.info("stacktrain base disk build ends.")
Example #6
0
def base_disk_exists():
    return vm.disk_exists(conf.get_base_disk_name())
Example #7
0
def vm_create_node(vm_name):

    try:
        vm_config = conf.vm[vm_name]
    except Exception:
        logger.exception("Failed to import VM configuration config.vm_%s.", vm_name)
        raise

    base_disk_name = conf.get_base_disk_name()

    vm.vm_delete(vm_name)

    libvirt_connect_uri = "qemu:///system"
    virt_install_call = ["sudo", "virt-install", "--connect={}".format(libvirt_connect_uri)]
    call_args = virt_install_call + [
        "--name",
        vm_name,
        "--ram",
        str(vm_config.vm_mem),
        "--vcpus",
        str(vm_config.vm_cpus),
        "--os-type={}".format("linux"),
        "--import",
    ]

    for index, iface in enumerate(conf.vm[vm_name].net_ifs):
        if iface["typ"] == "dhcp":
            call_args.extend(["--network", "bridge=virbr0"])
        elif iface["typ"] == "manual":
            net_name = "labs-{}".format(vm.ip_to_netname(iface["ip"]))
            call_args.extend(["--network", "network={}".format(net_name)])
        elif iface["typ"] == "static":
            net_name = "labs-{}".format(vm.ip_to_netname(iface["ip"]))
            call_args.extend(["--network", "network={}".format(net_name)])
        else:
            logger.error("Unknown interface type: %s", iface.typ)
            sys.exit(1)

    for index, disk in enumerate(conf.vm[vm_name].disks):
        # Turn number into letter (0->a, 1->b, etc.)
        disk_letter = chr(index + ord("a"))
        if disk is None:
            continue
        disk_name = "{}-sd{}".format(vm_name, disk_letter)
        if disk == "base":
            logger.info("Creating copy-on-write VM disk.")
            vm.disk_create_cow(disk_name, base_disk_name)
        else:
            size = disk
            logger.info("Adding empty disk to %s: %s", vm_name, disk_name)
            vm.disk_create(disk_name, size)
        call_args.extend(["--disk", "vol={}/{},cache=none".format(vm.kvm_vol_pool, disk_name)])

    if conf.vm_ui == "headless":
        call_args.extend(("--graphics", "none", "--noautoconsole"))
        call_args.append("--noreboot")
    elif conf.vm_ui == "vnc":
        # Only local connections allowed (0.0.0.0 would allow external
        # connections as well)
        call_args.extend(("--graphics", "vnc,listen=127.0.0.1"))
        call_args.append("--noreboot")
    # Default UI uses virt-viewer which doesn't fly with --noreboot

    import subprocess

    errout = subprocess.STDOUT
    logger.debug("virt-install call: %s", call_args)
    vm.virsh_log(call_args)
    subprocess.Popen(call_args, stderr=errout)

    # Prevent "time stamp from the future" due to race between two sudos for
    # virt-install (background) above and virsh below
    import time

    time.sleep(1)

    logger.info("Waiting for VM %s to be defined.", vm_name)
    while True:
        if vm.virsh_vm_defined(vm_name):
            logger.debug("VM %s is defined now.", vm_name)
            vm.log_xml_dump(vm_name, "defined")
            break
        time.sleep(1)
        print(".", end="")
        sys.stdout.flush()

    # Set VM group in description so we know which VMs are ours
    # (not with virt-install because older versions give an error for
    # --metadata title=TITLE)
    vm.set_vm_group(vm_name)
    vm.log_xml_dump(vm_name, "in_group")
Example #8
0
def main():
    abort_if_root_user()

    configure_logging()
    logger = logging.getLogger(__name__)

    logger.debug("Call args: %s", sys.argv)
    logger.debug(report.get_git_info())
    args = parse_args()
    set_conf_vars(args)

    import stacktrain.core.autostart as autostart
    import stacktrain.core.node_builder as node_builder
    import stacktrain.core.functions_host as host

    logger.info('%s(): caller: %s()', log_utils.get_fname(1),
                log_utils.get_fname(2))
    log_utils.log_message("this is a test")
    log_utils.log_entry()

    #sys.exit(1)

    # W0612: variable defined but not used
    # pylint_: disable=W0612
    # Only for the benefit of sfood
    # import stacktrain.virtualbox.install_base

    logger.debug("importing stacktrain.%s.install_base", conf.provider)
    install_base = importlib.import_module("stacktrain.%s.install_base" %
                                           conf.provider)
    logger.debug("importing stacktrain.%s.vm_create", conf.provider)
    vm = importlib.import_module("stacktrain.%s.vm_create" % conf.provider)

    vm.init()

    logger.info("stacktrain start at %s", time.strftime("%c"))

    # OS X sets LC_CTYPE to UTF-8 which results in errors when exported to
    # (remote) environments
    if "LC_CTYPE" in os.environ:
        logger.debug("Removing LC_CTYPE from environment.")
        del os.environ["LC_CTYPE"]
    # To be on the safe side, ensure a sane locale
    os.environ["LC_ALL"] = "C"

    logger.debug("Environment %s", os.environ)

    autostart.autostart_reset()

    if conf.wbatch:
        wbatch.wbatch_reset()

    if conf.do_build and not conf.leave_vms_running:
        vm.stop_running_cluster_vms()

    if conf.do_build and install_base.base_disk_exists():
        if args.target == "basedisk":
            print("Basedisk exists: %s" % conf.get_base_disk_name())
            print("\tDestroy and recreate? [y/N] ", end='')
            ans = raw_input().lower()
            if ans == 'y':
                logger.info("Deleting existing basedisk.")
                start_time = time.time()
                install_base.vm_install_base()
                logger.info("Basedisk build took %s seconds",
                            hf.fmt_time_diff(start_time))
            elif conf.wbatch:
                logger.info("Windows batch file build only.")
                tmp_do_build = conf.do_build
                conf.do_build = False
                install_base.vm_install_base()
                conf.do_build = tmp_do_build
            else:
                print("Nothing to do.")
            print("Done, returning now.")
            return
        elif conf.wbatch:
            logger.info("Windows batch file build only.")
            tmp_do_build = conf.do_build
            conf.do_build = False
            install_base.vm_install_base()
            conf.do_build = tmp_do_build
    else:
        start_time = time.time()
        install_base.vm_install_base()
        logger.info("Basedisk build took %s seconds",
                    hf.fmt_time_diff(start_time))

    if args.target == "basedisk":
        print("We are done.")
        return

    host.create_host_networks()

    start_time = time.time()
    node_builder.build_nodes(args.target)
    logger.info("Cluster build took %s seconds", hf.fmt_time_diff(start_time))

    report.print_summary()
Example #9
0
def vm_install_base():
    vm_name = "base"

    conf.vm[vm_name] = conf.VMconfig(vm_name)

    base_disk_name = conf.get_base_disk_name()

    vm.vm_delete(vm_name)
    vm.disk_delete(base_disk_name)

    vm_config = conf.vm[vm_name]

    if conf.do_build:
        install_iso = iso_image.find_install_iso()
    else:
        install_iso = os.path.join(conf.img_dir, conf.iso_image.name)

    logger.info("Install ISO:\n\t%s", install_iso)

    autostart.autostart_reset()

    autostart.autostart_queue("osbash/base_fixups.sh")

    autostart.autostart_from_config(conf.base_install_scripts)

    autostart.autostart_queue("zero_empty.sh", "shutdown.sh")

    base_disk_size = 10000
    vm.disk_create(base_disk_name, base_disk_size)

    libvirt_connect_uri = "qemu:///system"
    virt_install_call = ["sudo", "virt-install",
                         "--connect={}".format(libvirt_connect_uri)]
    vm_base_mem = 512

    call_args = virt_install_call
    call_args.extend(["--name", vm_name])
    call_args.extend(["--ram", str(vm_base_mem)])
    call_args.extend(["--vcpus", str(1)])
    call_args.extend(["--os-type", "linux"])
    call_args.extend(["--cdrom", install_iso])

    call_args.extend(["--disk", "vol={}/{},cache=none".format(vm.kvm_vol_pool,
                                                              base_disk_name)])

    if conf.vm_ui == "headless":
        call_args.extend(("--graphics", "none", "--noautoconsole"))
    elif conf.vm_ui == "vnc":
        call_args.extend(("--graphics", "vnc,listen=127.0.0.1"))
    # Default (no extra argument) is gui option: should open a console viewer
    call_args.append("--wait=-1")

    import subprocess
    errout = subprocess.STDOUT
    logger.debug("virt-install call: %s", ' '.join(call_args))
    logger.debug("virt-install call: %s", call_args)
    vm.virsh_log(call_args)
    subprocess.Popen(call_args, stderr=errout)

    while True:
        if vm.vm_is_running(vm_name):
            break
        print('.', end='')
        time.sleep(1)

    delay = 5
    logger.info("\nWaiting %d seconds for VM %s to come up.", delay, vm_name)
    cs.conditional_sleep(delay)

    logger.info("Booting into distribution installer.")
    distro_boot.distro_start_installer(vm_config)

    # Prevent "time stamp from the future" due to race between two sudos for
    # virt-install (background) above and virsh below
    time.sleep(1)

    logger.info("Waiting for VM %s to be defined.", vm_name)
    while True:
        if vm.vm_is_running(vm_name):
            break
        time.sleep(1)
        print(".")

    ssh_ip = vm.node_to_ip(vm_name)
    conf.vm[vm_name].ssh_ip = ssh_ip

    logger.info("Waiting for ping returning from %s.", ssh_ip)
    hf.wait_for_ping(ssh_ip)

    autostart.autostart_and_wait(vm_name)

    vm.vm_wait_for_shutdown(vm_name)

    logger.info("Compacting %s.", base_disk_name)
    vm.disk_compress(base_disk_name)

    vm.virsh("undefine", vm_name)

    del conf.vm[vm_name]

    logger.info("Base disk created.")

    logger.info("stacktrain base disk build ends.")
Example #10
0
def base_disk_exists():
    return vm.disk_exists(conf.get_base_disk_name())
Example #11
0
def main():
    abort_if_root_user()

    configure_logging()
    logger = logging.getLogger(__name__)

    logger.debug("Call args: %s", sys.argv)
    args = parse_args()
    set_conf_vars(args)

    import stacktrain.core.autostart as autostart
    import stacktrain.core.node_builder as node_builder
    import stacktrain.core.functions_host as host

    # W0612: variable defined but not used
    # pylint_: disable=W0612
    # Only for the benefit of sfood
    # import stacktrain.virtualbox.install_base

    logger.debug("importing stacktrain.%s.install_base", conf.provider)
    install_base = importlib.import_module("stacktrain.%s.install_base" % conf.provider)
    logger.debug("importing stacktrain.%s.vm_create", conf.provider)
    vm = importlib.import_module("stacktrain.%s.vm_create" % conf.provider)

    vm.init()

    logger.info("stacktrain start at %s", time.strftime("%c"))

    # OS X sets LC_CTYPE to UTF-8 which results in errors when exported to
    # (remote) environments
    if "LC_CTYPE" in os.environ:
        logger.debug("Removing LC_CTYPE from environment.")
        del os.environ["LC_CTYPE"]
    # To be on the safe side, ensure a sane locale
    os.environ["LC_ALL"] = "C"

    logger.debug("Environment %s", os.environ)

    autostart.autostart_reset()

    if conf.wbatch:
        wbatch.wbatch_reset()

    if conf.do_build and install_base.base_disk_exists():
        if args.target == "basedisk":
            print("Basedisk exists: %s" % conf.get_base_disk_name())
            print("\tDestroy and recreate? [y/N] ", end="")
            ans = raw_input().lower()
            if ans == "y":
                logger.info("Deleting existing basedisk.")
                start_time = time.time()
                install_base.vm_install_base()
                logger.info("Basedisk build took %s seconds", hf.fmt_time_diff(start_time))
            elif conf.wbatch:
                logger.info("Windows batch file build only.")
                tmp_do_build = conf.do_build
                conf.do_build = False
                install_base.vm_install_base()
                conf.do_build = tmp_do_build
            else:
                print("Nothing to do.")
            print("Done, returning now.")
            return
        elif conf.wbatch:
            logger.info("Windows batch file build only.")
            tmp_do_build = conf.do_build
            conf.do_build = False
            install_base.vm_install_base()
            conf.do_build = tmp_do_build
    else:
        start_time = time.time()
        install_base.vm_install_base()
        logger.info("Basedisk build took %s seconds", hf.fmt_time_diff(start_time))

    if args.target == "basedisk":
        print("We are done.")
        return

    host.create_host_networks()

    start_time = time.time()
    node_builder.build_nodes(args.target)
    logger.info("Cluster build took %s seconds", hf.fmt_time_diff(start_time))

    report.print_summary()
Example #12
0
def vm_create_node(vm_name):

    try:
        vm_config = conf.vm[vm_name]
    except Exception:
        logger.exception("Failed to import VM configuration config.vm_%s.",
                         vm_name)
        raise

    base_disk_name = conf.get_base_disk_name()

    vm.vm_delete(vm_name)

    libvirt_connect_uri = "qemu:///system"
    virt_install_call = [
        "sudo", "virt-install", "--connect={}".format(libvirt_connect_uri)
    ]
    call_args = virt_install_call + [
        "--name", vm_name, "--ram",
        str(vm_config.vm_mem), "--vcpus",
        str(vm_config.vm_cpus), "--os-type={}".format("linux"), "--import"
    ]

    for index, iface in enumerate(conf.vm[vm_name].net_ifs):
        if iface["typ"] == "dhcp":
            call_args.extend(["--network", "bridge=virbr0"])
        elif iface["typ"] == "manual":
            net_name = "labs-{}".format(vm.ip_to_netname(iface["ip"]))
            call_args.extend(["--network", "network={}".format(net_name)])
        elif iface["typ"] == "static":
            net_name = "labs-{}".format(vm.ip_to_netname(iface["ip"]))
            call_args.extend(["--network", "network={}".format(net_name)])
        else:
            logger.error("Unknown interface type: %s", iface.typ)
            sys.exit(1)

    for index, disk in enumerate(conf.vm[vm_name].disks):
        # Turn number into letter (0->a, 1->b, etc.)
        disk_letter = chr(index + ord('a'))
        if disk is None:
            continue
        disk_name = "{}-sd{}".format(vm_name, disk_letter)
        if disk == "base":
            logger.info("Creating copy-on-write VM disk.")
            vm.disk_create_cow(disk_name, base_disk_name)
        else:
            size = disk
            logger.info("Adding empty disk to %s: %s", vm_name, disk_name)
            vm.disk_create(disk_name, size)
        call_args.extend([
            "--disk", "vol={}/{},cache=none".format(vm.kvm_vol_pool, disk_name)
        ])

    if conf.vm_ui == "headless":
        call_args.extend(("--graphics", "none", "--noautoconsole"))
        call_args.append("--noreboot")
    elif conf.vm_ui == "vnc":
        # Only local connections allowed (0.0.0.0 would allow external
        # connections as well)
        call_args.extend(("--graphics", "vnc,listen=127.0.0.1"))
        call_args.append("--noreboot")
    # Default UI uses virt-viewer which doesn't fly with --noreboot

    import subprocess
    errout = subprocess.STDOUT
    logger.debug("virt-install call: %s", call_args)
    vm.virsh_log(call_args)
    subprocess.Popen(call_args, stderr=errout)

    # Prevent "time stamp from the future" due to race between two sudos for
    # virt-install (background) above and virsh below
    import time
    time.sleep(1)

    logger.info("Waiting for VM %s to be defined.", vm_name)
    while True:
        if vm.virsh_vm_defined(vm_name):
            logger.debug("VM %s is defined now.", vm_name)
            vm.log_xml_dump(vm_name, "defined")
            break
        time.sleep(1)
        print(".", end='')
        sys.stdout.flush()

    # Set VM group in description so we know which VMs are ours
    # (not with virt-install because older versions give an error for
    # --metadata title=TITLE)
    vm.set_vm_group(vm_name)
    vm.log_xml_dump(vm_name, "in_group")