def render_vlans(markdown):
    lines = []
    vlans = Vlan.objects().all()
    for vlan in vlans:
        vlan_id = vlan.vlan_id
        ip_range = vlan.ip_range
        netmask = vlan.netmask
        gateway = vlan.gateway
        ip_free = vlan.ip_free
        owner = vlan.owner
        rt_ticket = vlan.ticket
        cloud_name = ""
        try:
            if vlan.cloud:
                cloud_name = Cloud.objects(id=vlan.cloud.id).first().name
        except DoesNotExist:
            pass

        columns = [
            vlan_id,
            ip_range.strip(","),
            netmask,
            gateway,
            ip_free,
            owner,
            rt_ticket,
            cloud_name,
        ]

        lines.append(columns)

    for line in sorted(lines, key=lambda _line: _line[1]):
        entry = "| %s |\n" % " | ".join([str(col) for col in line])
        markdown.write(entry)
Beispiel #2
0
def main(_args):
    if _args.yaml:
        vlans = None
        try:
            with open(_args.yaml, "r") as _vlans_read:
                try:
                    vlans = yaml.safe_load(_vlans_read)
                except yaml.YAMLError:
                    logger.error("quads: Invalid YAML config: " + _args.yaml)
                    exit(1)
        except IOError as ex:
            logger.debug(ex)
            logger.error("Error reading %s" % _args.yaml)
            exit(1)
        if vlans:
            for vlan, properties in vlans.items():
                vlan_obj = Vlan.objects(vlan_id=properties["vlanid"]).first()
                cloud_name = "cloud0%s" % properties["cloud"] if int(properties["cloud"]) < 10 \
                    else "cloud%s" % properties["cloud"]
                cloud_obj = Cloud.objects(name=cloud_name).first()
                result, data = Vlan.prep_data(properties)
                if not result:
                    if cloud_obj:
                        data["cloud"] = cloud_obj
                    else:
                        data.pop("cloud")
                else:
                    logger.error("Failed to validate all fields")
                    exit(1)
                if vlan_obj:
                    vlan_obj.update(**data)
                    logger.info("Updated vlan: %s" % data["vlan_id"])
                else:
                    Vlan(**data).save()
                    logger.info("Inserted vlan: %s" % data["vlan_id"])
Beispiel #3
0
 def GET(self, **data):
     _args = {}
     if "date" in data:
         date = datetime.datetime.strptime(data["date"],
                                           "%Y-%m-%dt%H:%M:%S")
         _args["date"] = date
     if "host" in data:
         host = Host.objects(name=data["host"]).first()
         if host:
             _args["host"] = host
         else:
             return json.dumps({
                 "result":
                 ["Couldn't find host %s on Quads DB." % data["host"]]
             })
     if "cloud" in data:
         cloud = Cloud.objects(name=data["cloud"]).first()
         if cloud:
             _args["cloud"] = cloud
     if self.name == "current_schedule":
         _schedule = self.model.current_schedule(**_args)
         if _schedule:
             return _schedule.to_json()
         else:
             return json.dumps({"result": ["No results."]})
     return self.model.objects(**_args).to_json()
Beispiel #4
0
def create_initial_message(real_owner, cloud, cloud_info, ticket, cc):
    _cloud_obj = Cloud.objects(name=cloud).first()
    template_file = "initial_message"
    irc_bot_ip = conf["ircbot_ipaddr"]
    irc_bot_port = conf["ircbot_port"]
    irc_bot_channel = conf["ircbot_channel"]
    cc_users = conf["report_cc"].split(",")
    for user in cc:
        cc_users.append("%s@%s" % (user, conf["domain"]))
    if conf["email_notify"]:
        with open(os.path.join(TEMPLATES_PATH, template_file)) as _file:
            template = Template(_file.read())
        content = template.render(
            cloud_info=cloud_info,
            wp_wiki=conf["wp_wiki"],
            cloud=cloud,
            quads_url=conf["quads_url"],
            real_owner=real_owner,
            ticket=ticket,
            foreman_url=conf["foreman_url"],
        )

        postman = Postman("New QUADS Assignment Allocated", real_owner,
                          cc_users, content)
        postman.send_email()
    if conf["irc_notify"]:
        try:
            with Netcat(irc_bot_ip, irc_bot_port) as nc:
                message = "%s QUADS: %s is now active, choo choo! - http://%s/assignments/#%s" % (
                    irc_bot_channel, cloud_info, conf["wp_wiki"], cloud)
                nc.write(bytes(message.encode("utf-8")))
        except (TypeError, BrokenPipeError) as ex:
            logger.debug(ex)
            logger.error("Beep boop netcat can't communicate with your IRC.")
    _cloud_obj.update(notified=True)
def main(_args, _loop):
    clouds = Cloud.objects(validated=False, provisioned=True, name__ne="cloud01")
    for _cloud in clouds:
        _schedule_count = Schedule.current_schedule(cloud=_cloud).count()
        if _schedule_count and _cloud.wipe:
            validator = Validator(_cloud, _args, _loop=_loop)
            try:
                _loop.run_until_complete(validator.validate_env())
            except Exception as ex:
                logger.debug(ex)
                logger.info("Failed validation for %s" % _cloud.name)
Beispiel #6
0
async def main(_loop):
    no_extend_label = "CANNOT_EXTEND"
    extend_label = "CAN_EXTEND"

    try:
        jira = Jira(
            Config["jira_url"],
            loop=_loop,
        )
    except JiraException as ex:
        logger.error(ex)
        return 1

    tickets = await jira.get_pending_tickets()
    for ticket in tickets["issues"]:
        ticket_key = ticket.get("key").split("-")[-1]
        fields = ticket.get("fields")
        if fields:
            description = fields.get("description")
            try:
                cloud_field = description.split("\n")[1]
                cloud = cloud_field.split()[-1]
            except IndexError:
                logger.warning(
                    f"Could not retrieve cloud name from ticket {ticket_key}")

            cloud_obj = Cloud.objects(name=cloud).first()
            schedules = Schedule.current_schedule(cloud=cloud_obj)
            conflict = False
            for schedule in schedules:
                end_date = schedule.end + timedelta(weeks=2)
                available = Schedule.is_host_available(host=schedule.host.name,
                                                       start=schedule.end,
                                                       end=end_date)
                if not available:
                    conflict = True
                    await jira.add_label(ticket_key, no_extend_label)
                    logger.info(f"{cloud} labeled {no_extend_label}")
                    break

            if not conflict:
                await jira.add_label(ticket_key, extend_label)
                logger.info(f"{cloud} labeled {extend_label}")

            parent = fields.get("parent")
            if parent:
                p_ticket_key = parent.get("key").split("-")[-1]
                watchers = await jira.get_watchers(p_ticket_key)
                for watcher in watchers["watchers"]:
                    await jira.add_watcher(ticket_key, watcher["key"])
    return 0
Beispiel #7
0
def main() -> None:
    """
    Main function
    :return: None
    """
    cloud = Cloud.objects(name="cloud01").first()
    hosts = Host.objects(cloud=cloud, retired=False, broken=False)
    for host in hosts:
        file_name = f"{host.name}.xml"
        file_path = os.path.join(LSHW_OUTPUT_DIR, file_name)
        if os.path.exists(file_path):
            if os.path.getsize(file_path) < 1:
                run_lshw(host.name, file_path)
        else:
            run_lshw(host.name, file_path)
Beispiel #8
0
def verify(cloud):
    _cloud_obj = Cloud.objects(name=cloud).first()
    logger.info(f"Cloud qinq: {_cloud_obj.qinq}")
    if not _cloud_obj:
        logger.error(f"Cloud not found.")
        return

    if args.all:
        hosts = Host.objects(cloud=_cloud_obj)
    else:
        hosts = [Host.objects(cloud=_cloud_obj).first()]

    for host in hosts:
        if args.all:
            logger.info(f"{host.name}:")
        if host and host.interfaces:
            interfaces = sorted(host.interfaces, key=lambda k: k["name"])
            for i, interface in enumerate(interfaces):
                ssh_helper = SSHHelper(interface.switch_ip,
                                       Config["junos_username"])
                try:
                    _, vlan_member_out = ssh_helper.run_cmd(
                        "show configuration vlans | display set | match %s.0" %
                        interface.switch_port)
                    vlan_member = vlan_member_out[0].split()[2][4:].strip(",")
                except IndexError:
                    logger.warning(
                        "Could not determine the previous VLAN member for %s, switch %s, switch port %s "
                        % (
                            interface.name,
                            interface.switch_ip,
                            interface.switch_port,
                        ))
                    vlan_member = 0

                ssh_helper.disconnect()

                logger.info(
                    f"Interface em{i+1} appears to be a member of VLAN {vlan_member}",
                )

        else:
            logger.error(
                f"The cloud has no hosts or the host has no interfaces defined"
            )
Beispiel #9
0
def report_detailed(_logger, _start, _end):
    start = _start.replace(hour=21, minute=59, second=0)
    start_defer = start - timedelta(weeks=1)
    start_defer_id = date_to_object_id(start_defer)
    end = _end.replace(hour=22, minute=1, second=0)
    end_id = date_to_object_id(end)
    cloud_history = CloudHistory.objects(__raw__={
        "_id": {
            "$lt": end_id,
            "$gt": start_defer_id,
        },
    }).order_by("-_id")

    headers = [
        "Owner",
        "Ticket",
        "Cloud",
        "Description",
        "Systems",
        "Scheduled",
        "Duration",
    ]
    _logger.info(f"{headers[0]:<9}| "
                 f"{headers[1]:>9}| "
                 f"{headers[2]:>8}| "
                 f"{headers[3]:>10}| "
                 f"{headers[4]:>5}| "
                 f"{headers[5]:>10}| "
                 f"{headers[6]:>5}| ")

    for cloud in cloud_history:
        cloud_ref = Cloud.objects(name=cloud.name).first()
        schedule = Schedule.objects(
            Q(end__lt=end) & Q(start__gt=start)
            & Q(cloud=cloud_ref)).order_by("-_id")
        if schedule:
            delta = schedule[0].end - schedule[0].start
            description = cloud.description[:len(headers[3])]
            _logger.info(f"{cloud.owner:<9}| "
                         f"{cloud.ticket:>9}| "
                         f"{cloud.name:>8}| "
                         f"{description:>11}| "
                         f"{schedule.count():>7}| "
                         f"{str(schedule[0].start)[:10]:>9}| "
                         f"{delta.days:>8}| ")
Beispiel #10
0
def render_vlans(markdown):
    lines = []
    vlans = Vlan.objects().all()
    for vlan in vlans:
        cloud_obj = Cloud.objects(vlan=vlan).first()
        vlan_id = vlan.vlan_id
        ip_range = vlan.ip_range
        netmask = vlan.netmask
        gateway = vlan.gateway
        ip_free = vlan.ip_free
        cloud_current_count = Schedule.current_schedule(
            cloud=cloud_obj).count()
        if cloud_obj and cloud_current_count > 0:
            owner = cloud_obj.owner
            ticket = cloud_obj.ticket
            cloud_name = cloud_obj.name
        else:
            owner = "nobody"
            ticket = ""
            cloud_name = ""

        columns = [
            vlan_id,
            ip_range.strip(","),
            netmask,
            gateway,
            ip_free,
            owner,
            ticket,
            cloud_name,
        ]

        lines.append(columns)

    for line in sorted(lines, key=lambda _line: _line[1]):
        entry = "| %s |\n" % " | ".join([str(col) for col in line])
        markdown.write(entry)
Beispiel #11
0
def main():
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    future_days = 7

    _all_clouds = Cloud.objects()
    _active_clouds = [
        _cloud for _cloud in _all_clouds
        if Schedule.current_schedule(cloud=_cloud).count() > 0
    ]
    _validated_clouds = [
        _cloud for _cloud in _active_clouds if _cloud.validated
    ]

    if not os.path.exists(os.path.join(conf["data_dir"], "report")):
        Path(os.path.join(conf["data_dir"], "report")).mkdir(parents=True,
                                                             exist_ok=True)

    for cloud in _validated_clouds:
        notification_obj = Notification.objects(cloud=cloud,
                                                ticket=cloud.ticket).first()
        current_hosts = Schedule.current_schedule(cloud=cloud)
        cloud_info = "%s: %s (%s)" % (
            cloud.name,
            current_hosts.count(),
            cloud.description,
        )
        if not notification_obj.initial:
            logger.info("=============== Initial Message")
            loop.run_until_complete(
                create_initial_message(
                    cloud.owner,
                    cloud.name,
                    cloud_info,
                    cloud.ticket,
                    cloud.ccuser,
                ))
            notification_obj.update(initial=True)

        for day in Days:
            future = datetime.now() + timedelta(days=day.value)
            future_date = "%4d-%.2d-%.2d 22:00" % (
                future.year,
                future.month,
                future.day,
            )
            future_hosts = Schedule.current_schedule(cloud=cloud,
                                                     date=future_date)

            diff = set(current_hosts) - set(future_hosts)
            if diff and future > current_hosts[0].end:
                if not notification_obj[
                        day.name.lower()] and conf["email_notify"]:
                    logger.info("=============== Additional Message")
                    host_list = [schedule.host.name for schedule in diff]
                    create_message(
                        cloud,
                        day.value,
                        cloud_info,
                        host_list,
                    )
                    kwargs = {day.name.lower(): True}
                    notification_obj.update(**kwargs)
                    break

    for cloud in _all_clouds:
        notification_obj = Notification.objects(cloud=cloud,
                                                ticket=cloud.ticket).first()
        if cloud.name != "cloud01" and cloud.owner not in ["quads", None]:
            current_hosts = Schedule.current_schedule(cloud=cloud)
            cloud_info = "%s: %s (%s)" % (
                cloud.name,
                current_hosts.count(),
                cloud.description,
            )

            if not notification_obj.pre_initial and conf["email_notify"]:
                logger.info("=============== Future Initial Message")
                create_future_initial_message(
                    cloud,
                    cloud_info,
                )
                notification_obj.update(pre_initial=True)

            for day in range(1, future_days + 1):
                if not notification_obj.pre and cloud.validated:
                    future = datetime.now() + timedelta(days=day)
                    future_date = "%4d-%.2d-%.2d 22:00" % (
                        future.year,
                        future.month,
                        future.day,
                    )
                    future_hosts = Schedule.current_schedule(cloud=cloud,
                                                             date=future_date)

                    if future_hosts.count() > 0:
                        diff = set(current_hosts) - set(future_hosts)
                        host_list = [schedule.host.name for schedule in diff]
                        if diff:
                            logger.info("=============== Additional Message")
                            create_future_message(
                                cloud,
                                day,
                                cloud_info,
                                host_list,
                            )
                            notification_obj.update(pre=True)
                            break
Beispiel #12
0
def switch_config(host, old_cloud, new_cloud):
    _host_obj = Host.objects(name=host).first()
    _old_cloud_obj = Cloud.objects(name=old_cloud).first()
    _new_cloud_obj = Cloud.objects(name=new_cloud).first()
    if not _host_obj.interfaces:
        logger.error("Host has no interfaces defined.")
        return False
    logger.debug("Connecting to switch on: %s" %
                 _host_obj.interfaces[0].switch_ip)
    switch_ip = None
    ssh_helper = None
    interfaces = sorted(_host_obj.interfaces, key=lambda k: k["name"])
    for i, interface in enumerate(interfaces):
        last_nic = i == len(_host_obj.interfaces) - 1
        if not switch_ip:
            switch_ip = interface.switch_ip
            try:
                ssh_helper = SSHHelper(switch_ip, Config["junos_username"])
            except SSHHelperException:
                logger.error(f"Failed to connect to switch: {switch_ip}")
                return False
        else:
            if switch_ip != interface.switch_ip:
                ssh_helper.disconnect()
                switch_ip = interface.switch_ip
                ssh_helper = SSHHelper(switch_ip, Config["junos_username"])
        result, old_vlan_out = ssh_helper.run_cmd(
            "show configuration interfaces %s" % interface.switch_port)
        old_vlan = None
        if result and old_vlan_out:
            old_vlan = old_vlan_out[0].split(";")[0].split()[1][7:]
        if not old_vlan:
            if not _new_cloud_obj.vlan and not last_nic:
                logger.warning(
                    "Warning: Could not determine the previous VLAN for %s on %s, switch %s, switchport %s"
                    % (
                        host,
                        interface.name,
                        interface.switch_ip,
                        interface.switch_port,
                    ))
            old_vlan = get_vlan(_old_cloud_obj, i)

        new_vlan = get_vlan(_new_cloud_obj, i)

        if _new_cloud_obj.vlan and last_nic:
            if int(old_vlan) != int(_new_cloud_obj.vlan.vlan_id):
                logger.info("Setting last interface to public vlan %s." %
                            new_vlan)

                juniper = Juniper(
                    interface.switch_ip,
                    interface.switch_port,
                    old_vlan,
                    _new_cloud_obj.vlan.vlan_id,
                )
                success = juniper.convert_port_public()

                if success:
                    logger.info("Successfully updated switch settings.")
                else:
                    logger.error(
                        "There was something wrong updating switch for %s:%s" %
                        (host, interface.name))
                    return False
        else:
            if int(old_vlan) != int(new_vlan):
                juniper = Juniper(interface.switch_ip, interface.switch_port,
                                  old_vlan, new_vlan)
                success = juniper.set_port()

                if success:
                    logger.info("Successfully updated switch settings.")
                else:
                    logger.error(
                        "There was something wrong updating switch for %s:%s" %
                        (host, interface.name))
                    return False

    if ssh_helper:
        ssh_helper.disconnect()

    return True
Beispiel #13
0
async def move_and_rebuild(host,
                           new_cloud,
                           semaphore,
                           rebuild=False,
                           loop=None):
    build_start = datetime.now()
    logger.debug("Moving and rebuilding host: %s" % host)

    untouchable_hosts = Config["untouchable_hosts"]
    logger.debug("Untouchable hosts: %s" % untouchable_hosts)
    _host_obj = Host.objects(name=host).first()

    if host in untouchable_hosts:
        logger.error("No way...")
        return False

    _target_cloud = Cloud.objects(name=new_cloud).first()

    ipmi_new_pass = (f"{Config['infra_location']}@{_target_cloud.ticket}"
                     if _target_cloud.ticket else Config["ipmi_password"])

    ipmi_set_pass = [
        "user",
        "set",
        "password",
        str(Config["ipmi_cloud_username_id"]),
        ipmi_new_pass,
    ]

    new_semaphore = asyncio.Semaphore(20)
    await execute_ipmi(host, arguments=ipmi_set_pass, semaphore=new_semaphore)

    ipmi_set_operator = [
        "user", "priv",
        str(Config["ipmi_cloud_username_id"]), "0x4"
    ]
    await execute_ipmi(host,
                       arguments=ipmi_set_operator,
                       semaphore=new_semaphore)

    badfish = None
    if rebuild and _target_cloud.name != _host_obj.default_cloud.name:
        if Config.pdu_management:
            # TODO: pdu management
            pass

        try:
            badfish = await badfish_factory(
                "mgmt-%s" % host,
                Config["ipmi_username"],
                Config["ipmi_password"],
                propagate=True,
            )
        except BadfishException:
            logger.error(
                f"Could not initialize Badfish. Verify ipmi credentials for mgmt-{host}."
            )
            return False

        if is_supported(host):
            try:
                interfaces_path = os.path.join(
                    os.path.dirname(__file__),
                    "../../conf/idrac_interfaces.yml")
                await badfish.change_boot("director", interfaces_path)

                # wait 10 minutes for the boot order job to complete
                await asyncio.sleep(600)
            except BadfishException:
                logger.error(
                    f"Could not set boot order via Badfish for mgmt-{host}.")
                return False

        try:
            await badfish.set_power_state("on")
        except BadfishException:
            logger.error(f"Failed to power on {host}")
            return False
        foreman_results = []
        params = [
            {
                "name": "operatingsystems",
                "value": Config["foreman_default_os"],
                "identifier": "title",
            },
            {
                "name": "ptables",
                "value": Config["foreman_default_ptable"]
            },
            {
                "name": "media",
                "value": Config["foreman_default_medium"]
            },
        ]

        foreman = Foreman(
            Config["foreman_api_url"],
            Config["foreman_username"],
            Config["foreman_password"],
            semaphore=semaphore,
            loop=loop,
        )

        set_result = await foreman.set_host_parameter(host, "overcloud",
                                                      "true")
        foreman_results.append(set_result)

        put_result = await foreman.put_parameter(host, "build", 1)
        foreman_results.append(put_result)

        put_param_result = await foreman.put_parameters_by_name(host, params)
        foreman_results.append(put_param_result)

        owner_id = await foreman.get_user_id(new_cloud)
        host_id = await foreman.get_host_id(host)
        put_result = await foreman.put_element("hosts", host_id, "owner_id",
                                               owner_id)
        foreman_results.append(put_result)

        for result in foreman_results:
            if isinstance(result, Exception) or not result:
                logger.error(
                    "There was something wrong setting Foreman host parameters."
                )
                return False
        if is_supported(host):
            try:
                await badfish.boot_to_type(
                    "foreman",
                    os.path.join(os.path.dirname(__file__),
                                 "../../conf/idrac_interfaces.yml"),
                )
                await badfish.reboot_server(graceful=False)
            except BadfishException:
                logger.error(f"Error setting PXE boot via Badfish on {host}.")
                await badfish.reboot_server(graceful=False)
                return False
        else:
            try:
                asyncio.run_coroutine_threadsafe(
                    badfish.unmount_virtual_media(),
                    loop,
                )
            except BadfishException:
                logger.warning(
                    f"Could not unmount virtual media for mgmt-{host}.")

            try:
                ipmi_pxe_persistent = [
                    "chassis",
                    "bootdev",
                    "pxe",
                    "options=persistent",
                ]
                await execute_ipmi(host,
                                   arguments=ipmi_pxe_persistent,
                                   semaphore=new_semaphore)
                await ipmi_reset(host, new_semaphore)
            except Exception as ex:
                logger.debug(ex)
                logger.error(
                    f"There was something wrong setting PXE flag or resetting IPMI on {host}."
                )

    if _target_cloud.name == _host_obj.default_cloud.name:
        if not badfish:
            try:
                badfish = await badfish_factory(
                    "mgmt-%s" % host,
                    Config["ipmi_username"],
                    Config["ipmi_password"],
                    propagate=True,
                )
            except BadfishException:
                logger.error(
                    f"Could not initialize Badfish. Verify ipmi credentials for mgmt-{host}."
                )
                return False

        await badfish.set_power_state("off")
        source_cloud_schedule = Schedule.current_schedule(
            cloud=_host_obj.cloud.name)
        if not source_cloud_schedule:
            _old_cloud_obj = Cloud.objects(name=_host_obj.cloud.name).first()
            _old_cloud_obj.update(vlan=None)

    schedule = Schedule.current_schedule(cloud=_target_cloud,
                                         host=_host_obj).first()
    if schedule:
        schedule.update(build_start=build_start, build_end=datetime.now())
        schedule.save()

    logger.debug("Updating host: %s")
    _host_obj.update(cloud=_target_cloud,
                     build=False,
                     last_build=datetime.now(),
                     validated=False)
    return True
Beispiel #14
0
    ssh_helper.disconnect()

    return True


def validate_env(_cloud):
    if not post_system_test(_cloud):
        if env_allocation_time_exceeded(_cloud):
            notify_failure(_cloud)
            return

    if not post_network_test(_cloud):
        if env_allocation_time_exceeded(_cloud):
            notify_failure(_cloud)
            return

    # TODO: gather ansible-cmdb facts

    # TODO: quads dell config report

    notify_success(_cloud)
    _cloud.update(validated=True)
    return


if __name__ == "__main__":
    clouds = Cloud.objects(released=True, validated=False, name__ne="cloud01")
    for cloud in clouds:
        validate_env(cloud)
Beispiel #15
0
def make_env_json(filename):
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    foreman = Foreman(
        conf["foreman_api_url"],
        conf["foreman_username"],
        conf["foreman_password"],
        loop=loop,
    )

    cloud_list = Cloud.objects()

    if not os.path.exists(conf["json_web_path"]):
        os.makedirs(conf["json_web_path"])

    now = time.time()
    old_jsons = [
        file for file in os.listdir(conf["json_web_path"]) if ":" in file
    ]
    for file in old_jsons:
        if (os.stat(os.path.join(conf["json_web_path"], file)).st_mtime <
                now - conf["json_retention_days"] * 86400):
            os.remove(os.path.join(conf["json_web_path"], file))

    for cloud in cloud_list:
        host_list = Host.objects(cloud=cloud).order_by("name")

        foreman_password = conf["ipmi_password"]
        if cloud.ticket:
            foreman_password = f"{conf['infra_location']}@{cloud.ticket}"

        data = defaultdict(list)
        for host in host_list:
            if conf["foreman_unavailable"]:
                overcloud = {"result": "true"}
            else:
                overcloud = loop.run_until_complete(
                    foreman.get_host_param(host.name, "overcloud"))

            if not overcloud:
                overcloud = {"result": "true"}

            if type(overcloud["result"]) != bool:
                try:
                    _overcloud_result = strtobool(overcloud["result"])
                except ValueError:
                    print(
                        f"WARN: {host.name} overcloud value is not set correctly."
                    )
                    _overcloud_result = 1
            else:
                _overcloud_result = overcloud["result"]

            if "result" in overcloud and _overcloud_result:
                mac = []
                if filename == "instackenv":
                    for interface in host.interfaces:
                        if interface.pxe_boot:
                            mac.append(interface.mac_address)
                if filename == "ocpinventory":
                    mac = [
                        interface.mac_address for interface in host.interfaces
                    ]
                data["nodes"].append({
                    "pm_password": foreman_password,
                    "pm_type": "pxe_ipmitool",
                    "mac": mac,
                    "cpu": "2",
                    "memory": "1024",
                    "disk": "20",
                    "arch": "x86_64",
                    "pm_user": conf["ipmi_cloud_username"],
                    "pm_addr": "mgmt-%s" % host.name,
                })

        content = json.dumps(data, indent=4, sort_keys=True)

        if not os.path.exists(conf["json_web_path"]):
            pathlib.Path(conf["json_web_path"]).mkdir(parents=True,
                                                      exist_ok=True)

        now = datetime.now()
        new_json_file = os.path.join(
            conf["json_web_path"],
            "%s_%s.json_%s" %
            (cloud.name, filename, now.strftime("%Y-%m-%d_%H:%M:%S")),
        )
        json_file = os.path.join(conf["json_web_path"],
                                 "%s_%s.json" % (cloud.name, filename))
        with open(new_json_file, "w+") as _json_file:
            _json_file.seek(0)
            _json_file.write(content)
        os.chmod(new_json_file, 0o644)
        copyfile(new_json_file, json_file)
Beispiel #16
0
async def move_and_rebuild(host,
                           new_cloud,
                           semaphore,
                           rebuild=False,
                           loop=None):
    build_start = datetime.now()
    logger.debug("Moving and rebuilding host: %s" % host)

    untouchable_hosts = conf["untouchable_hosts"]
    logger.debug("Untouchable hosts: %s" % untouchable_hosts)
    _host_obj = Host.objects(name=host).first()

    if host in untouchable_hosts:
        logger.error("No way...")
        return False

    _new_cloud_obj = Cloud.objects(name=new_cloud).first()

    ipmi_new_pass = (f"{conf['infra_location']}@{_new_cloud_obj.ticket}"
                     if _new_cloud_obj.ticket else conf["ipmi_password"])

    ipmi_set_pass = [
        "user",
        "set",
        "password",
        str(conf["ipmi_cloud_username_id"]),
        ipmi_new_pass,
    ]

    new_semaphore = asyncio.Semaphore(20)
    await execute_ipmi(host, arguments=ipmi_set_pass, semaphore=new_semaphore)

    ipmi_set_operator = [
        "user", "priv",
        str(conf["ipmi_cloud_username_id"]), "0x4"
    ]
    await execute_ipmi(host,
                       arguments=ipmi_set_operator,
                       semaphore=new_semaphore)

    if rebuild and _new_cloud_obj.name != _host_obj.default_cloud.name:
        if "pdu_management" in conf and conf["pdu_management"]:
            # TODO: pdu management
            pass

        if is_supported(host):
            try:
                badfish = await badfish_factory(
                    "mgmt-%s" % host,
                    conf["ipmi_username"],
                    conf["ipmi_password"],
                    propagate=True,
                )
            except BadfishException:
                logger.error(
                    f"Could not initialize Badfish. Verify ipmi credentials for mgmt-{host}."
                )
                return False
            try:
                changed_boot_order = asyncio.run_coroutine_threadsafe(
                    badfish.change_boot(
                        "director",
                        os.path.join(os.path.dirname(__file__),
                                     "../../conf/idrac_interfaces.yml"),
                    ),
                    loop,
                )
                if changed_boot_order:
                    await badfish.reboot_server(graceful=False)
            except BadfishException:
                logger.error(
                    f"Could not set boot order via Badfish for mgmt-{host}.")
                return False

        foreman_results = []
        params = [
            {
                "name": "operatingsystems",
                "value": conf["foreman_default_os"],
                "identifier": "title",
            },
            {
                "name": "ptables",
                "value": conf["foreman_default_ptable"]
            },
            {
                "name": "media",
                "value": conf["foreman_default_medium"]
            },
        ]

        foreman = Foreman(
            conf["foreman_api_url"],
            conf["foreman_username"],
            conf["foreman_password"],
            semaphore=semaphore,
            loop=loop,
        )

        set_result = await foreman.set_host_parameter(host, "overcloud",
                                                      "true")
        foreman_results.append(set_result)

        put_result = await foreman.put_parameter(host, "build", 1)
        foreman_results.append(put_result)

        put_param_result = await foreman.put_parameters_by_name(host, params)
        foreman_results.append(put_param_result)

        owner_id = await foreman.get_user_id(new_cloud)
        host_id = await foreman.get_host_id(host)
        put_result = await foreman.put_element("hosts", host_id, "owner_id",
                                               owner_id)
        foreman_results.append(put_result)

        for result in foreman_results:
            if isinstance(result, Exception) or not result:
                logger.error(
                    "There was something wrong setting Foreman host parameters."
                )
                return False

        healthy = False
        for i in range(RETRIES):
            nc = Netcat(_host_obj.name)
            if nc.health_check():
                healthy = True
                nc.close()
                break
            nc.close()

        if not healthy:
            logger.error("Failed to recover host after changing boot order.")
            return False

        if is_supported(host):
            try:
                await badfish.boot_to_type(
                    "foreman",
                    os.path.join(os.path.dirname(__file__),
                                 "../../conf/idrac_interfaces.yml"),
                )
                await badfish.reboot_server(graceful=False)
            except BadfishException:
                logger.error(f"Error setting PXE boot via Badfish on {host}.")
                await badfish.reboot_server(graceful=False)
                return False
        else:
            try:
                ipmi_pxe_persistent = [
                    "chassis",
                    "bootdev",
                    "pxe",
                    "options",
                    "=",
                    "persistent",
                ]
                await execute_ipmi(host,
                                   arguments=ipmi_pxe_persistent,
                                   semaphore=new_semaphore)
                await ipmi_reset(host, new_semaphore)
            except Exception as ex:
                logger.debug(ex)
                logger.error(
                    f"There was something wrong setting PXE flag or resetting IPMI on {host}."
                )

    schedule = Schedule.current_schedule(cloud=_new_cloud_obj,
                                         host=_host_obj).first()
    if schedule:
        schedule.update(build_start=build_start, build_end=datetime.now())
        schedule.save()

    logger.debug("Updating host: %s")
    _host_obj.update(cloud=_new_cloud_obj,
                     build=False,
                     last_build=datetime.now(),
                     validated=False)
    return True
Beispiel #17
0
    def GET(self, **data):
        args = {}
        _cloud = None
        _host = None
        if "cloudonly" in data:
            _cloud = Cloud.objects(cloud=data["cloudonly"])
            if not _cloud:
                cherrypy.response.status = "404 Not Found"
                return json.dumps(
                    {"result": "Cloud %s Not Found" % data["cloudonly"]})
            else:
                return _cloud.to_json()

        if self.name == "host":
            if "id" in data:
                _host = Host.objects(id=data["id"]).first()
            elif "name" in data:
                _host = Host.objects(name=data["name"]).first()
            elif "cloud" in data:
                _cloud = Cloud.objects(name=data["cloud"]).first()
                _host = Host.objects(cloud=_cloud)
            else:
                _host = Host.objects()
            if not _host:
                return json.dumps({"result": ["Nothing to do."]})
            return _host.to_json()

        if self.name == "ccuser":
            _clouds = self.model.objects().all()
            clouds_summary = []
            for cloud in _clouds:
                count = Schedule.current_schedule(cloud=cloud).count()
                clouds_summary.append({
                    "name": cloud.name,
                    "count": count,
                    "description": cloud.description,
                    "owner": cloud.owner,
                    "ticket": cloud.ticket,
                    "ccuser": cloud.ccuser,
                    "provisioned": cloud.provisioned,
                })

            return json.dumps(clouds_summary)

        if self.name == "cloud":
            if "id" in data:
                _cloud = Cloud.objects(id=data["id"]).first()
            elif "name" in data:
                _cloud = Cloud.objects(name=data["name"]).first()
            elif "owner" in data:
                _cloud = Cloud.to_json(owner=data["owner"]).first()
            if _cloud:
                return _cloud.to_json()

        if self.name == "available":

            _start = _end = datetime.datetime.now()
            if "start" in data:
                _start = datetime.datetime.strptime(data["start"],
                                                    "%Y-%m-%dT%H:%M:%S")

            if "end" in data:
                _end = datetime.datetime.strptime(data["end"],
                                                  "%Y-%m-%dT%H:%M:%S")

            available = []
            all_hosts = Host.objects().all()

            for host in all_hosts:
                if Schedule.is_host_available(host=host["name"],
                                              start=_start,
                                              end=_end):
                    available.append(host.name)
            return json.dumps(available)

        if self.name == "summary":
            _clouds = Cloud.objects().all()
            clouds_summary = []
            total_count = 0
            for cloud in _clouds:
                if cloud.name == "cloud01":
                    count = Host.objects(cloud=cloud,
                                         retired=False,
                                         broken=False).count()
                else:
                    date = datetime.datetime.now()
                    if "date" in data:
                        date = datetime.datetime.strptime(
                            data["date"], "%Y-%m-%dT%H:%M:%S")
                    count = self.model.current_schedule(cloud=cloud,
                                                        date=date).count()
                    total_count += count
                clouds_summary.append({
                    "name": cloud.name,
                    "count": count,
                    "description": cloud.description,
                    "owner": cloud.owner,
                    "ticket": cloud.ticket,
                    "ccuser": cloud.ccuser,
                    "provisioned": cloud.provisioned,
                    "validated": cloud.validated,
                })
            if "date" in data:
                host_count = Host.objects(retired=False, broken=False).count()
                for cloud in clouds_summary:
                    if cloud["name"] == "cloud01":
                        cloud["count"] = host_count - total_count

            return json.dumps(clouds_summary)

        if self.name == "qinq":
            _clouds = Cloud.objects().all()
            clouds_qinq = []
            for cloud in _clouds:
                _type = "Isolated"
                if cloud.qinq == 1:
                    _type = "Combined"
                qinq_value = f"{cloud.qinq} ({_type})"
                clouds_qinq.append({"name": cloud.name, "qinq": qinq_value})

            return json.dumps(clouds_qinq)

        if self.name == "broken":
            _hosts = self.model.objects(broken=True)
            broken = []
            for host in _hosts:
                broken.append(host.name)

            return json.dumps(broken)

        if self.name == "retired":
            hosts = [host.name for host in self.model.objects(retired=True)]
            return json.dumps(hosts)

        objs = self.model.objects(**args)
        if objs:
            return objs.to_json()
        else:
            return json.dumps({"result": ["No results."]})
Beispiel #18
0
        return


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Validate Quads assignments")
    parser.add_argument(
        "--debug",
        action="store_true",
        default=False,
        help="Show debugging information.",
    )
    args = parser.parse_args()

    level = logging.INFO
    if args.debug:
        level = logging.DEBUG

    logging.basicConfig(level=level, format="%(message)s")

    clouds = Cloud.objects(validated=False, provisioned=True, name__ne="cloud01")
    for _cloud in clouds:
        _schedule_count = Schedule.current_schedule(cloud=_cloud).count()
        if _schedule_count and _cloud.wipe:
            validator = Validator(_cloud)
            try:
                validator.validate_env()
            except Exception as ex:
                logger.debug(ex)
                logger.info("Failed validation for %s" % _cloud.name)
Beispiel #19
0
def verify(_cloud_name, change=False):
    _cloud_obj = Cloud.objects(name=_cloud_name).first()
    hosts = Host.objects(cloud=_cloud_obj)

    for _host_obj in hosts:
        logger.info(f"Host: {_host_obj.name}")
        if _host_obj.interfaces:
            interfaces = sorted(_host_obj.interfaces, key=lambda k: k["name"])
            for i, interface in enumerate(interfaces):
                ssh_helper = SSHHelper(interface.ip_address,
                                       conf["junos_username"])
                last_nic = i == len(_host_obj.interfaces) - 1
                vlan = get_vlan(_cloud_obj, i, last_nic)

                try:
                    old_vlan_out = ssh_helper.run_cmd(
                        "show configuration interfaces %s" %
                        interface.switch_port)
                    old_vlan = old_vlan_out[0].split(";")[0].split()[1]
                    if old_vlan.startswith("QinQ"):
                        old_vlan = old_vlan[7:]
                except IndexError:
                    old_vlan = None

                try:
                    vlan_member_out = ssh_helper.run_cmd(
                        "show configuration vlans | display set | match %s.0" %
                        interface.switch_port)
                    vlan_member = vlan_member_out[0].split()[2][4:].strip(",")
                except IndexError:
                    if not _cloud_obj.vlan and not last_nic:
                        logger.warning(
                            "Could not determine the previous VLAN member for %s, switch %s, switch port %s "
                            % (
                                interface.name,
                                interface.ip_address,
                                interface.switch_port,
                            ))
                    vlan_member = None

                ssh_helper.disconnect()

                if not old_vlan:
                    if not _cloud_obj.vlan and not last_nic:
                        logger.warning(
                            "Could not determine the previous VLAN for %s, switch %s, switchport %s"
                            % (
                                interface.name,
                                interface.ip_address,
                                interface.switch_port,
                            ))
                    old_vlan = get_vlan(_cloud_obj, i, last_nic)

                if int(old_vlan) != int(vlan):
                    logger.warning("Interface %s not using QinQ_vl%s",
                                   interface.switch_port, vlan)

                if _cloud_obj.vlan and i == len(_host_obj.interfaces) - 1:
                    vlan = _cloud_obj.vlan.vlan_id

                if not vlan_member or int(vlan_member) != int(vlan):
                    if not _cloud_obj.vlan and not last_nic:
                        logger.warning(
                            "Interface %s appears to be a member of VLAN %s, should be %s",
                            interface.switch_port,
                            vlan_member,
                            vlan,
                        )

                    if change:
                        if _cloud_obj.vlan and last_nic:
                            if int(_cloud_obj.vlan.vlan_id) != int(old_vlan):

                                logger.info(
                                    f"Change requested for {interface.name}")
                                logger.info(
                                    "Setting last interface to public vlan %s."
                                    % _cloud_obj.vlan.vlan_id)

                                success = juniper_convert_port_public(
                                    interface.ip_address,
                                    interface.switch_port,
                                    old_vlan,
                                    vlan,
                                )
                            else:
                                logger.info(
                                    f"No changes required for {interface.name}"
                                )
                                continue
                        else:
                            logger.info(
                                f"Change requested for {interface.name}")
                            success = juniper_set_port(
                                interface.ip_address,
                                interface.switch_port,
                                vlan_member,
                                vlan,
                            )

                        if success:
                            logger.info(
                                "Successfully updated switch settings.")
                        else:
                            logger.error(
                                f"There was something wrong updating switch for {interface.name}"
                            )
Beispiel #20
0
def main():
    if conf["openstack_management"]:
        foreman = Foreman(conf["foreman_api_url"], conf["foreman_username"],
                          conf["foreman_password"])

        cloud_list = Cloud.objects()

        if not os.path.exists(conf["json_web_path"]):
            os.makedirs(conf["json_web_path"])

        now = time.time()
        old_jsons = [
            file for file in os.listdir(conf["json_web_path"]) if ":" in file
        ]
        for file in old_jsons:
            if os.stat(os.path.join(conf["json_web_path"], file)
                       ).st_mtime < now - conf["json_retention_days"] * 86400:
                os.remove(os.path.join(conf["json_web_path"], file))

        for cloud in cloud_list:
            host_list = Host.objects(cloud=cloud).order_by("name")

            foreman_password = conf["ipmi_password"]
            if cloud.ticket:
                foreman_password = cloud.ticket

            json_data = defaultdict(list)
            for host in host_list[1:]:
                if conf["foreman_unavailable"]:
                    overcloud = {"result": "true"}
                else:
                    overcloud = foreman.get_host_param(host.name, "overcloud")
                if not overcloud:
                    overcloud = {"result": "true"}
                if "result" in overcloud and strtobool(overcloud["result"]):
                    mac = "00:00:00:00:00:00"
                    if len(host.interfaces) > 1:
                        mac = host.interfaces[1].mac_address
                    json_data['nodes'].append({
                        'pm_password':
                        foreman_password,
                        'pm_type':
                        "pxe_ipmitool",
                        'mac': [mac],
                        'cpu':
                        "2",
                        'memory':
                        "1024",
                        'disk':
                        "20",
                        'arch':
                        "x86_64",
                        'pm_user':
                        conf["ipmi_cloud_username"],
                        'pm_addr':
                        "mgmt-%s" % host.name
                    })

            content = json.dumps(json_data, indent=4, sort_keys=True)

            if not os.path.exists(conf["json_web_path"]):
                pathlib.Path(conf["json_web_path"]).mkdir(parents=True,
                                                          exist_ok=True)

            now = datetime.now()
            new_json_file = os.path.join(
                conf["json_web_path"], "%s_instackenv.json_%s" %
                (cloud.name, now.strftime("%Y-%m-%d_%H:%M:%S")))
            json_file = os.path.join(conf["json_web_path"],
                                     "%s_instackenv.json" % cloud.name)
            with open(new_json_file, "w+") as _json_file:
                _json_file.seek(0)
                _json_file.write(content)
            os.chmod(new_json_file, 0o644)
            copyfile(new_json_file, json_file)
Beispiel #21
0
def main():
    days = [1, 3, 5, 7]
    future_days = 7

    _all_clouds = Cloud.objects()
    _active_clouds = [
        _cloud for _cloud in _all_clouds
        if Schedule.current_schedule(cloud=_cloud).count() > 0
    ]
    _validated_clouds = [
        _cloud for _cloud in _active_clouds if _cloud.validated
    ]

    if not os.path.exists(os.path.join(conf["data_dir"], "report")):
        Path(os.path.join(conf["data_dir"], "report")).mkdir(parents=True,
                                                             exist_ok=True)

    for cloud in _validated_clouds:
        current_hosts = Schedule.current_schedule(cloud=cloud)
        cloud_info = "%s: %s (%s)" % (cloud.name, current_hosts.count(),
                                      cloud.description)
        if not cloud["notified"]:
            logger.info('=============== Initial Message')
            create_initial_message(
                cloud.owner,
                cloud.name,
                cloud_info,
                cloud.ticket,
                cloud.ccuser,
            )

        for day in days:
            future = datetime.now() + timedelta(days=day)
            future_date = "%4d-%.2d-%.2d 22:00" % (future.year, future.month,
                                                   future.day)
            future_hosts = Schedule.current_schedule(cloud=cloud,
                                                     date=future_date)

            diff = set(current_hosts) - set(future_hosts)
            if diff and future < current_hosts[0].end:
                report_file = "%s-%s-%s-%s" % (cloud.name, cloud.owner, day,
                                               cloud.ticket)
                report_path = os.path.join(conf["data_dir"], "report",
                                           report_file)
                if not os.path.exists(report_path) and conf["email_notify"]:
                    logger.info('=============== Additional Message')
                    host_list = [schedule.host.name for schedule in diff]
                    create_message(
                        cloud.owner,
                        day,
                        cloud.name,
                        cloud_info,
                        cloud.ccuser,
                        host_list,
                        report_path,
                    )
                    break

    for cloud in _all_clouds:
        if cloud.name != "cloud01" and cloud.owner not in ["quads", None]:
            current_hosts = Schedule.current_schedule(cloud=cloud)
            cloud_info = "%s: %s (%s)" % (cloud.name, current_hosts.count(),
                                          cloud.description)

            report_pre_ini_file = "%s-%s-pre-initial-%s" % (
                cloud.name, cloud.owner, cloud.ticket)
            report_pre_ini_path = os.path.join(conf["data_dir"], "report",
                                               report_pre_ini_file)
            if not os.path.exists(
                    report_pre_ini_path) and conf["email_notify"]:
                logger.info('=============== Future Initial Message')
                create_future_initial_message(
                    cloud.owner,
                    cloud_info,
                    cloud.ccuser,
                    report_pre_ini_path,
                )

            report_pre_file = "%s-%s-pre-%s" % (cloud.name, cloud.owner,
                                                cloud.ticket)
            report_pre_path = os.path.join(conf["data_dir"], "report",
                                           report_pre_file)
            if not os.path.exists(report_pre_path) and cloud.validated:
                future = datetime.now() + timedelta(days=future_days)
                future_date = "%4d-%.2d-%.2d 22:00" % (
                    future.year, future.month, future.day)
                future_hosts = Schedule.current_schedule(cloud=cloud,
                                                         date=future_date)

                diff = set(current_hosts) - set(future_hosts)
                host_list = [schedule.host.name for schedule in diff]
                if diff:
                    logger.info('=============== Additional Message')
                    create_future_message(
                        cloud.owner,
                        future_days,
                        cloud.name,
                        cloud_info,
                        cloud.ccuser,
                        host_list,
                        report_pre_path,
                    )
Beispiel #22
0
def verify(_cloud_name, change=False):
    _cloud_obj = Cloud.objects(name=_cloud_name).first()
    quads = Api(API_URL)

    hosts = quads.get_cloud_hosts(_cloud_name)

    for _host in hosts:
        _host_obj = Host.objects(name=_host["name"]).first()
        if _host_obj.interfaces:

            _public_vlan_obj = Vlan.objects(cloud=_cloud_obj).first()

            for i, interface in enumerate(_host_obj.interfaces):
                ssh_helper = SSHHelper(interface.ip_address,
                                       conf["junos_username"])
                vlan = get_vlan(_cloud_obj, i)

                try:
                    old_vlan_out = ssh_helper.run_cmd(
                        "show configuration interfaces %s" %
                        interface.switch_port)
                    old_vlan = old_vlan_out[0].split(";")[0].split()[1][7:]
                except IndexError:
                    old_vlan = None

                try:
                    vlan_member_out = ssh_helper.run_cmd(
                        "show vlans interface %s.0" % interface.switch_port)
                    vlan_member = vlan_member_out[1].split()[2][4:].strip(",")
                except IndexError:
                    vlan_member = None

                ssh_helper.disconnect()

                if not old_vlan:
                    logger.warning(
                        "Could not determine the previous VLAN for %s on %s, switch %s, switchport %s"
                        % (_host["name"], interface.name, interface.ip_address,
                           interface.switch_port))
                    old_vlan = get_vlan(_cloud_obj, i)

                if not vlan_member:
                    logger.warning(
                        "Could not determine the previous VLAN member for %s on %s, switch %s, switchport %s"
                        % (_host["name"], interface.name, interface.ip_address,
                           interface.switch_port))
                    vlan_member = old_vlan

                if int(old_vlan) != int(vlan):
                    logger.warning("interface %s not using QinQ_vl%s",
                                   interface.switch_port, vlan)

                if _public_vlan_obj and i == len(_host_obj.interfaces) - 1:
                    vlan = _public_vlan_obj.vlan_id

                if int(vlan_member) != int(vlan):
                    logger.warning(
                        "interface %s appears to be a member of VLAN %s, should be %s",
                        interface.switch_port, vlan_member, vlan)

                    if change:
                        logger.info('=== INFO: change requested')

                        if _public_vlan_obj and i == len(
                                _host_obj.interfaces) - 1:
                            logger.info(
                                "Setting last interface to public vlan %s." %
                                _public_vlan_obj.vlan_id)

                            success = juniper_convert_port_public(
                                interface.ip_address, interface.switch_port,
                                str(old_vlan), str(vlan))
                        else:
                            success = juniper_set_port(interface.ip_address,
                                                       interface.switch_port,
                                                       str(vlan_member),
                                                       str(vlan))

                        if success:
                            logger.info(
                                "Successfully updated switch settings.")
                        else:
                            logger.error(
                                "There was something wrong updating switch for %s:%s"
                                % (_host.name, interface.name))
Beispiel #23
0
def move_and_rebuild(host, old_cloud, new_cloud, rebuild=False):
    logger.debug("Moving and rebuilding host: %s" % host)

    untouchable_hosts = conf["untouchable_hosts"]
    logger.debug("Untouchable hosts: %s" % untouchable_hosts)
    _host_obj = Host.objects(name=host).first()
    if not _host_obj.interfaces:
        logger.error("Host has no interfaces defined.")
        return False

    if host in untouchable_hosts:
        logger.error("No way...")
        return False

    _old_cloud_obj = Cloud.objects(name=old_cloud).first()
    _new_cloud_obj = Cloud.objects(name=new_cloud).first()

    logger.debug("Connecting to switch on: %s" %
                 _host_obj.interfaces[0].ip_address)
    _public_vlan_obj = Vlan.objects(cloud=_new_cloud_obj).first()
    for i, interface in enumerate(_host_obj.interfaces):
        ssh_helper = SSHHelper(interface.ip_address, conf["junos_username"])
        old_vlan_out = ssh_helper.run_cmd("show configuration interfaces %s" %
                                          interface.switch_port)
        old_vlan = old_vlan_out[0].split(";")[0].split()[1][7:]
        if not old_vlan:
            logger.warning(
                "Warning: Could not determine the previous VLAN for %s on %s, switch %s, switchport %s"
                % (host, interface.name, interface.ip_address,
                   interface.switch_port))
            old_vlan = get_vlan(_old_cloud_obj, i)

        new_vlan = get_vlan(_new_cloud_obj, i)

        if _public_vlan_obj and i == len(_host_obj.interfaces) - 1:
            logger.info("Setting last interface to public vlan %s." % new_vlan)

            if int(old_vlan) != int(_public_vlan_obj.vlan_id):
                success = juniper_convert_port_public(
                    interface.ip_address, interface.switch_port, str(old_vlan),
                    str(_public_vlan_obj.vlan_id))
                if success:
                    logger.info("Successfully updated switch settings.")
                else:
                    logger.error(
                        "There was something wrong updating switch for %s:%s" %
                        (host, interface.name))
                    return False
        else:

            if int(old_vlan) != int(new_vlan):
                success = juniper_set_port(interface.ip_address,
                                           interface.switch_port,
                                           str(old_vlan), str(new_vlan))
                if success:
                    logger.info("Successfully update switch settings.")
                else:
                    logger.error(
                        "There was something wrong updating switch for %s:%s" %
                        (host, interface.name))
                    return False

        if i == len(_host_obj.interfaces) - 1:
            _old_vlan_obj = Vlan.objects(cloud=_old_cloud_obj).first()
            if _old_vlan_obj:
                _old_vlan_obj.update(cloud=None)

        ssh_helper.disconnect()

    ipmi_new_pass = _new_cloud_obj.ticket if _new_cloud_obj.ticket else conf[
        "$ipmi_password"]

    foreman = Foreman(
        conf["foreman_api_url"],
        conf["foreman_username"],
        conf["foreman_password"],
    )
    foreman.remove_role(_old_cloud_obj.name, _host_obj.name)
    foreman.add_role(_new_cloud_obj.name, _host_obj.name)
    foreman.update_user_password(_new_cloud_obj.name, ipmi_new_pass)

    ipmi_set_pass = [
        "/usr/bin/ipmitool", "-I", "lanplus", "-H",
        "mgmt-%s" % host, "-U", conf["ipmi_username"], "-P",
        conf["ipmi_password"], "user", "set", "password",
        str(conf["ipmi_cloud_username_id"]), ipmi_new_pass
    ]
    logger.debug("ipmi_set_pass: %s" % ipmi_set_pass)
    subprocess.call(ipmi_set_pass)

    ipmi_set_operator = [
        "/usr/bin/ipmitool", "-I", "lanplus", "-H",
        "mgmt-%s" % host, "-U", conf["ipmi_username"], "-P",
        conf["ipmi_password"], "user", "priv",
        str(conf["ipmi_cloud_username_id"]), "0x4"
    ]
    logger.debug("ipmi_set_operator: %s" % ipmi_set_operator)
    subprocess.call(ipmi_set_operator)
    if rebuild and _new_cloud_obj.name != "cloud01":
        badfish = Badfish("mgmt-%s" % host, conf["ipmi_username"],
                          conf["ipmi_password"])

        if "pdu_management" in conf and conf["pdu_management"]:
            # TODO: pdu management
            pass

        if is_supermicro(host):
            ipmi_pxe_persistent = [
                "/usr/bin/ipmitool", "-I", "lanplus", "-H",
                "mgmt-%s" % host, "-U", conf["ipmi_username"], "-P",
                conf["ipmi_password"], "chassis", "bootdev", "pxe", "options",
                "=", "persistent"
            ]
            logger.debug("ipmi_pxe_persistent: %s" % ipmi_pxe_persistent)
            subprocess.call(ipmi_pxe_persistent)

        if is_supported(host):
            try:
                badfish.change_boot(
                    "director",
                    os.path.join(os.path.dirname(__file__),
                                 "../../conf/idrac_interfaces.yml"))
            except SystemExit as ex:
                logger.debug(ex)
                logger.error("Could not set boot order via Badfish.")
                return False

        foreman_success = foreman.remove_extraneous_interfaces(host)

        foreman_success = foreman.set_host_parameter(
            host, "rhel73", "false") and foreman_success
        foreman_success = foreman.set_host_parameter(
            host, "rhel75", "false") and foreman_success
        foreman_success = foreman.set_host_parameter(
            host, "overcloud", "true") and foreman_success
        foreman_success = foreman.put_parameter(host, "build",
                                                1) and foreman_success
        foreman_success = foreman.put_parameter_by_name(
            host, "operatingsystems", conf["foreman_default_os"],
            "title") and foreman_success
        foreman_success = foreman.put_parameter_by_name(
            host, "ptables",
            conf["foreman_default_ptable"]) and foreman_success
        foreman_success = foreman.put_parameter_by_name(
            host, "media", conf["foreman_default_medium"]) and foreman_success
        if not foreman_success:
            logger.error(
                "There was something wrong setting Foreman host parameters.")

        try:
            badfish.set_next_boot_pxe()
            badfish.reboot_server()
        except SystemExit:
            if is_supermicro(host):
                logger.warning("Badfish not yet supported on Supermicro: %s." %
                               host)
            else:
                logger.exception(
                    "There was something wrong setting next PXE boot via Badfish."
                )
                return False

        logger.debug("Updating host: %s")
        _host_obj.update(cloud=_new_cloud_obj,
                         build=False,
                         last_build=datetime.now())
    else:

        logger.debug("Updating host: %s")
        _host_obj.update(cloud=_new_cloud_obj,
                         build=False,
                         last_build=datetime.now())
Beispiel #24
0
    ssh_helper.disconnect()

    return True


def validate_env(_cloud):
    if not post_system_test(_cloud):
        if env_allocation_time_exceeded(_cloud):
            notify_failure(_cloud)
            return

    if not post_network_test(_cloud):
        if env_allocation_time_exceeded(_cloud):
            notify_failure(_cloud)
            return

    # TODO: gather ansible-cmdb facts

    # TODO: quads dell config report

    notify_success(_cloud)
    _cloud.update(validated=True)
    return


if __name__ == "__main__":
    clouds = Cloud.objects(released=True, validated=False)
    for cloud in clouds:
        validate_env(cloud)
def verify(_cloud_name, _host_name, change=False):
    if not _cloud_name and not _host_name:
        logger.warning(
            f"At least one of --cloud or --host should be specified.")
        return

    _cloud_obj = None
    if _cloud_name:
        _cloud_obj = Cloud.objects(name=_cloud_name).first()

    if _host_name:
        hosts = Host.objects(name=_host_name, retired=False)
    else:
        hosts = Host.objects(cloud=_cloud_obj, retired=False)
    first_host = hosts.first()

    if not _cloud_obj:
        _cloud_obj = first_host.cloud

    if _cloud_obj != first_host.cloud:
        logger.warning(f"Both --cloud and --host have been specified.")
        logger.warning(f"Host: {first_host.name}")
        logger.warning(f"Cloud: {_cloud_obj.name}")
        logger.warning(
            f"However, {first_host.name} is a member of {first_host.cloud.name}"
        )
        logger.warning(f"!!!!! Be certain this is what you want to do. !!!!!")

    for _host_obj in hosts:
        logger.info(f"Host: {_host_obj.name}")
        if _host_obj.interfaces:
            interfaces = sorted(_host_obj.interfaces, key=lambda k: k["name"])
            for i, interface in enumerate(interfaces):
                ssh_helper = SSHHelper(interface.switch_ip,
                                       Config["junos_username"])
                last_nic = i == len(_host_obj.interfaces) - 1
                vlan = get_vlan(_cloud_obj, i, last_nic)

                try:
                    _, old_vlan_out = ssh_helper.run_cmd(
                        "show configuration interfaces %s" %
                        interface.switch_port)
                    old_vlan = old_vlan_out[0].split(";")[0].split()[1]
                    if old_vlan.startswith("QinQ"):
                        old_vlan = old_vlan[7:]
                except IndexError:
                    old_vlan = 0

                try:
                    _, vlan_member_out = ssh_helper.run_cmd(
                        "show configuration vlans | display set | match %s.0" %
                        interface.switch_port)
                    vlan_member = vlan_member_out[0].split()[2][4:].strip(",")
                except IndexError:
                    if not _cloud_obj.vlan and not last_nic:
                        logger.warning(
                            "Could not determine the previous VLAN member for %s, switch %s, switch port %s "
                            % (
                                interface.name,
                                interface.switch_ip,
                                interface.switch_port,
                            ))
                    vlan_member = 0

                ssh_helper.disconnect()

                if int(old_vlan) != int(vlan):
                    logger.warning("Interface %s not using QinQ_vl%s",
                                   interface.switch_port, vlan)

                if int(vlan_member) != int(vlan):
                    if not last_nic:
                        logger.warning(
                            "Interface %s appears to be a member of VLAN %s, should be %s",
                            interface.switch_port,
                            vlan_member,
                            vlan,
                        )

                    if change:
                        if _cloud_obj.vlan and last_nic:
                            if int(_cloud_obj.vlan.vlan_id) != int(old_vlan):

                                logger.info(
                                    f"Change requested for {interface.name}")
                                logger.info(
                                    "Setting last interface to public vlan %s."
                                    % _cloud_obj.vlan.vlan_id)

                                juniper = Juniper(
                                    interface.switch_ip,
                                    interface.switch_port,
                                    old_vlan,
                                    vlan,
                                )
                                success = juniper.convert_port_public()
                            else:
                                logger.info(
                                    f"No changes required for {interface.name}"
                                )
                                continue
                        else:
                            logger.info(
                                f"Change requested for {interface.name}")
                            juniper = Juniper(
                                interface.switch_ip,
                                interface.switch_port,
                                vlan_member,
                                vlan,
                            )
                            success = juniper.set_port()

                        if success:
                            logger.info(
                                "Successfully updated switch settings.")
                        else:
                            logger.error(
                                f"There was something wrong updating switch for {interface.name}"
                            )
Beispiel #26
0
    def POST(self, **data):
        # make sure post data passed in is ready to pass to mongo engine
        result, data = Schedule.prep_data(data)

        _start = None
        _end = None

        if "start" in data:
            _start = datetime.datetime.strptime(data["start"],
                                                "%Y-%m-%d %H:%M")

        if "end" in data:
            _end = datetime.datetime.strptime(data["end"], "%Y-%m-%d %H:%M")

        _host = data["host"]
        _host_obj = Host.objects(name=_host).first()

        broken_hosts = Host.objects(broken=True)
        if _host_obj in broken_hosts:
            result.append(f"Host {_host_obj.name} is in broken state")

        # Check if there were data validation errors
        if result:
            result = ["Data validation failed: %s" % ", ".join(result)]
            cherrypy.response.status = "400 Bad Request"
            return json.dumps({"result": result})

        cloud_obj = None
        if "cloud" in data:
            cloud_obj = Cloud.objects(name=data["cloud"]).first()
            if not cloud_obj:
                result.append("Provided cloud does not exist")
                cherrypy.response.status = "400 Bad Request"
                return json.dumps({"result": result})

        if "index" in data:
            data["host"] = _host_obj
            schedule = self.model.objects(index=data["index"],
                                          host=data["host"]).first()
            if schedule:
                if not _start:
                    _start = schedule.start
                if not _end:
                    _end = schedule.end
                if not cloud_obj:
                    cloud_obj = schedule.cloud
                if Schedule.is_host_available(host=_host,
                                              start=_start,
                                              end=_end,
                                              exclude=schedule.index):
                    data["cloud"] = cloud_obj
                    notification_obj = Notification.objects(
                        cloud=cloud_obj, ticket=cloud_obj.ticket).first()
                    if notification_obj:
                        notification_obj.update(
                            one_day=False,
                            three_days=False,
                            five_days=False,
                            seven_days=False,
                        )
                    schedule.update(**data)
                    result.append("Updated %s %s" %
                                  (self.name, schedule.index))
                else:
                    result.append(
                        "Host is not available during that time frame")
        else:
            try:
                if Schedule.is_host_available(host=_host,
                                              start=_start,
                                              end=_end):

                    if (self.model.current_schedule(cloud=cloud_obj)
                            and cloud_obj.validated):
                        if not cloud_obj.wipe:
                            _host_obj.update(validated=True)
                        notification_obj = Notification.objects(
                            cloud=cloud_obj, ticket=cloud_obj.ticket).first()
                        if notification_obj:
                            notification_obj.update(success=False)

                    schedule = Schedule()
                    data["cloud"] = cloud_obj
                    schedule.insert_schedule(**data)
                    cherrypy.response.status = "201 Resource Created"
                    result.append("Added schedule for %s on %s" %
                                  (data["host"], cloud_obj.name))
                else:
                    result.append(
                        "Host is not available during that time frame")

            except Exception as e:
                # TODO: make sure when this is thrown the output
                #       points back to here and gives the end user
                #       enough information to fix the issue
                cherrypy.response.status = "500 Internal Server Error"
                result.append("Error: %s" % e)
        return json.dumps({"result": result})
def main():
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    foreman = Foreman(
        conf["foreman_api_url"],
        conf["foreman_username"],
        conf["foreman_password"],
        loop=loop,
    )

    lines = []
    all_hosts = loop.run_until_complete(foreman.get_all_hosts())
    blacklist = re.compile("|".join(
        [re.escape(word) for word in conf["exclude_hosts"].split("|")]))

    broken_hosts = Host.objects(broken=True)
    domain_broken_hosts = [
        host for host in broken_hosts if conf["domain"] in host.name
    ]

    mgmt_hosts = {}
    for host, properties in all_hosts.items():
        if not blacklist.search(host):
            if properties.get("sp_name", False):
                properties["host_ip"] = all_hosts.get(host, {"ip": None})["ip"]
                properties["host_mac"] = all_hosts.get(host,
                                                       {"mac": None})["mac"]
                properties["ip"] = properties.get("sp_ip")
                properties["mac"] = properties.get("sp_mac")
                mgmt_hosts[properties.get("sp_name")] = properties

    lines.append("### **SUMMARY**\n")
    _summary = print_summary()
    lines.extend(_summary)
    details_header = ["\n", "### **DETAILS**\n", "\n"]
    lines.extend(details_header)
    summary_response = requests.get(os.path.join(API_URL, "summary"))
    _cloud_summary = []
    if summary_response.status_code == 200:
        _cloud_summary = summary_response.json()
    for cloud in [cloud for cloud in _cloud_summary if cloud["count"] > 0]:
        name = cloud["name"]
        owner = cloud["owner"]
        lines.append("### <a name=%s></a>\n" % name.strip())
        lines.append(
            "### **%s : %s (%s) -- %s**\n\n" %
            (name.strip(), cloud["count"], cloud["description"], owner))
        lines.extend(print_header())
        _cloud_obj = Cloud.objects(name=name).first()
        _hosts = sorted(
            Host.objects(cloud=_cloud_obj, retired=False, broken=False),
            key=lambda x: x.name,
        )
        for host in _hosts:
            lines.extend(add_row(host))
        lines.append("\n")

    lines.extend(print_unmanaged(mgmt_hosts))
    lines.extend(print_faulty(domain_broken_hosts))

    _full_path = os.path.join(conf["wp_wiki_git_repo_path"], "assignments.md")

    if not os.path.exists(conf["wp_wiki_git_repo_path"]):
        pathlib.Path(conf["wp_wiki_git_repo_path"]).mkdir(parents=True,
                                                          exist_ok=True)

    with open(_full_path, "w+") as _f:
        _f.seek(0)
        for cloud in lines:
            _line = cloud if cloud else ""
            _f.write(_line)

        _f.truncate()
def print_summary():
    _summary = []
    _headers = [
        "**NAME**",
        "**SUMMARY**",
        "**OWNER**",
        "**REQUEST**",
        '<span id="status">**STATUS**</span>',
    ]
    if conf["openstack_management"]:
        _headers.append("**OSPENV**")
    if conf["openshift_management"]:
        _headers.append("**OCPINV**")
    if conf["gather_ansible_facts"]:
        _headers.append("**HWFACTS**")

    _summary.append("| %s |\n" % " | ".join(_headers))
    _summary.append("| %s |\n" %
                    " | ".join(["---" for _ in range(len(_headers))]))

    _cloud_response = requests.get(os.path.join(API_URL, "summary"))
    _cloud_summary = []
    if _cloud_response.status_code == 200:
        _cloud_summary = _cloud_response.json()

    for cloud in [cloud for cloud in _cloud_summary if cloud["count"] > 0]:
        cloud_name = cloud["name"]
        desc = "%s (%s)" % (cloud["count"], cloud["description"])
        owner = cloud["owner"]
        ticket = cloud["ticket"]
        link = "<a href=%s/%s-%s target=_blank>%s</a>" % (
            conf["ticket_url"],
            conf["ticket_queue"],
            ticket,
            ticket,
        )
        cloud_specific_tag = "%s_%s_%s" % (cloud_name, owner, ticket)

        style_tag_end = "</span>"
        if cloud["validated"] or cloud_name == "cloud01":
            style_tag_start = '<span style="color:green">'
            instack_link = os.path.join(conf["quads_url"], "cloud",
                                        "%s_instackenv.json" % cloud_name)
            instack_text = "download"
            ocpinv_link = os.path.join(conf["quads_url"], "cloud",
                                       "%s_ocpinventory.json" % cloud_name)
            ocpinv_text = "download"
            status = (
                '<span class="progress" style="margin-bottom:0px"><span role="progressbar" aria-valuenow="100" '
                'aria-valuemin="0" aria-valuemax="100" style="width:100%" class="progress-bar">100%</span></span> '
            )
        else:
            cloud_obj = Cloud.objects(name=cloud_name).first()
            scheduled_hosts = Schedule.current_schedule(
                cloud=cloud_obj).count()
            moved_hosts = Host.objects(cloud=cloud_obj).count()
            percent = moved_hosts / scheduled_hosts * 100
            style_tag_start = '<span style="color:red">'
            instack_link = "#"
            instack_text = "validating"
            ocpinv_link = "#"
            ocpinv_text = "validating"
            if percent < 15:
                classes = [
                    "progress-bar",
                    "progress-bar-striped",
                    "progress-bar-danger",
                    "active",
                ]
                status = (
                    '<span class="progress" style="margin-bottom:0px"><span role="progressbar" '
                    'aria-valuenow="100" aria-valuemin="0" aria-valuemax="100" style="width:100%%" '
                    'class="%s">%.0f%%</span></span>' %
                    (" ".join(classes), percent))
            else:
                classes = [
                    "progress-bar",
                    "progress-bar-striped",
                    "progress-bar-warning",
                    "active",
                ]
                status = (
                    '<span class="progress" style="margin-bottom:0px"><span role="progressbar" '
                    'aria-valuenow="%.0f" aria-valuemin="0" aria-valuemax="100" style="width:%.0f%%" '
                    'class="%s">%.0f%%</span></span>' %
                    (percent, percent, " ".join(classes), percent))

        _data = [
            "[%s%s%s](#%s)" %
            (style_tag_start, cloud_name, style_tag_end, cloud_name),
            desc,
            owner,
            link,
        ]

        if conf["gather_ansible_facts"]:
            factstyle_tag_end = "</span>"
            if os.path.exists(
                    os.path.join(
                        conf["ansible_facts_web_path"],
                        "ansible_facts",
                        "%s_overview.html" % cloud_specific_tag,
                    )):
                factstyle_tag_start = '<span style="color:green">'
                ansible_facts_link = os.path.join(
                    conf["quads_url"],
                    "ansible_facts",
                    "%s_overview.html" % cloud_specific_tag,
                )
            else:
                factstyle_tag_start = '<span style="color:red">'
                ansible_facts_link = os.path.join(conf["quads_url"],
                                                  "underconstruction")
            if cloud_name == "cloud01":
                _data.append("")
                _data.append("")
                _data.append(status)
                _data.append("")
            else:
                _data.append("<a href=%s target=_blank>%s%s%s</a>" %
                             (instack_link, style_tag_start, instack_text,
                              style_tag_end))
                _data.append(
                    "<a href=%s target=_blank>%s%s%s</a>" %
                    (ocpinv_link, style_tag_start, ocpinv_text, style_tag_end))
                _data.append(status)
                _data.append("<a href=%s target=_blank>%sinventory%s</a>" %
                             (ansible_facts_link, factstyle_tag_start,
                              factstyle_tag_end))
        else:
            _data.append(status)
            if cloud_name == "cloud01":
                if conf["openstack_management"]:
                    _data.append("")
                if conf["openshift_management"]:
                    _data.append("")
            else:
                if conf["openstack_management"]:
                    _data.append("<a href=%s target=_blank>%s%s%s</a>" %
                                 (instack_link, style_tag_start, instack_text,
                                  style_tag_end))
                if conf["openshift_management"]:
                    _data.append("<a href=%s target=_blank>%s%s%s</a>" %
                                 (ocpinv_link, style_tag_start, ocpinv_text,
                                  style_tag_end))

        _summary.append("| %s |\n" % " | ".join(_data))

    _hosts = Host.objects(broken=False, retired=False)
    _host_count = len(_hosts)
    _schedules = Schedule.current_schedule().count()
    _daily_percentage = _schedules * 100 // _host_count
    _summary.append(f"| Total | {_host_count} |\n")
    _summary.append("\n")
    _summary.append(f"Daily Utilization: {_daily_percentage}% \n")
    _summary.append("\n")
    _summary.append("[Unmanaged Hosts](#unmanaged)\n")
    _summary.append("\n")
    _summary.append("[Faulty Hosts](#faulty)\n")

    return _summary
Beispiel #29
0
def main():
    loop = asyncio.get_event_loop()

    foreman_admin = Foreman(
        conf["foreman_api_url"],
        conf["foreman_username"],
        conf["foreman_password"],
        loop=loop,
    )

    ignore = ["cloud01"]
    foreman_rbac_exclude = conf.get("foreman_rbac_exclude")
    if foreman_rbac_exclude:
        ignore.extend(foreman_rbac_exclude.split("|"))
    clouds = Cloud.objects()
    for cloud in clouds:

        infra_pass = f"{conf['infra_location']}@{cloud.ticket}"
        loop.run_until_complete(
            foreman_admin.update_user_password(cloud.name, infra_pass)
        )

        foreman_cloud_user = Foreman(
            conf["foreman_api_url"],
            cloud.name,
            infra_pass,
            loop=loop,
        )

        if cloud.name not in ignore:
            logger.info(f"Processing {cloud.name}")

            cloud_hosts = loop.run_until_complete(foreman_cloud_user.get_all_hosts())

            user_id = loop.run_until_complete(foreman_admin.get_user_id(cloud.name))
            admin_id = loop.run_until_complete(
                foreman_admin.get_user_id(conf["foreman_username"])
            )

            current_schedule = Schedule.current_schedule(cloud=cloud)
            if current_schedule:

                logger.info(f"  Current Host Permissions:")
                for host, properties in cloud_hosts.items():
                    logger.info(f"    {host}")

                    match = [
                        schedule.host.name
                        for schedule in current_schedule
                        if schedule.host.name == host
                    ]
                    if not match:
                        _host_id = loop.run_until_complete(
                            foreman_admin.get_host_id(host)
                        )
                        loop.run_until_complete(
                            foreman_admin.put_element(
                                "hosts", _host_id, "owner_id", admin_id
                            )
                        )
                        logger.info(f"* Removed permission {host}")

                for schedule in current_schedule:
                    match = [
                        host
                        for host, _ in cloud_hosts.items()
                        if host == schedule.host.name
                    ]
                    if not match:
                        # want to run these separately to avoid ServerDisconnect
                        _host_id = loop.run_until_complete(
                            foreman_admin.get_host_id(schedule.host.name)
                        )
                        loop.run_until_complete(
                            foreman_admin.put_element(
                                "hosts", _host_id, "owner_id", user_id
                            )
                        )
                        logger.info(f"* Added permission {schedule.host.name}")
            else:
                if cloud_hosts:
                    logger.info("  No active schedule, removing pre-existing roles.")
                    for host, properties in cloud_hosts.items():
                        _host_id = loop.run_until_complete(
                            foreman_admin.get_host_id(host)
                        )
                        loop.run_until_complete(
                            foreman_admin.put_element(
                                "hosts", _host_id, "owner_id", admin_id
                            )
                        )
                        logger.info(f"* Removed permission {host}")
                else:
                    logger.info("  No active schedule nor roles assigned.")