def render_vlans(markdown):
lines = []
vlans = Vlan.objects().all()
for vlan in vlans:
vlan_id = vlan.vlan_id
ip_range = vlan.ip_range
netmask = vlan.netmask
gateway = vlan.gateway
ip_free = vlan.ip_free
owner = vlan.owner
rt_ticket = vlan.ticket
cloud_name = ""
try:
if vlan.cloud:
cloud_name = Cloud.objects(id=vlan.cloud.id).first().name
except DoesNotExist:
pass
columns = [
vlan_id,
ip_range.strip(","),
netmask,
gateway,
ip_free,
owner,
rt_ticket,
cloud_name,
]
lines.append(columns)
for line in sorted(lines, key=lambda _line: _line[1]):
entry = "| %s |\n" % " | ".join([str(col) for col in line])
markdown.write(entry)
def main(_args):
if _args.yaml:
vlans = None
try:
with open(_args.yaml, "r") as _vlans_read:
try:
vlans = yaml.safe_load(_vlans_read)
except yaml.YAMLError:
logger.error("quads: Invalid YAML config: " + _args.yaml)
exit(1)
except IOError as ex:
logger.debug(ex)
logger.error("Error reading %s" % _args.yaml)
exit(1)
if vlans:
for vlan, properties in vlans.items():
vlan_obj = Vlan.objects(vlan_id=properties["vlanid"]).first()
cloud_name = "cloud0%s" % properties["cloud"] if int(properties["cloud"]) < 10 \
else "cloud%s" % properties["cloud"]
cloud_obj = Cloud.objects(name=cloud_name).first()
result, data = Vlan.prep_data(properties)
if not result:
if cloud_obj:
data["cloud"] = cloud_obj
else:
data.pop("cloud")
else:
logger.error("Failed to validate all fields")
exit(1)
if vlan_obj:
vlan_obj.update(**data)
logger.info("Updated vlan: %s" % data["vlan_id"])
else:
Vlan(**data).save()
logger.info("Inserted vlan: %s" % data["vlan_id"])
def GET(self, **data):
_args = {}
if "date" in data:
date = datetime.datetime.strptime(data["date"],
"%Y-%m-%dt%H:%M:%S")
_args["date"] = date
if "host" in data:
host = Host.objects(name=data["host"]).first()
if host:
_args["host"] = host
else:
return json.dumps({
"result":
["Couldn't find host %s on Quads DB." % data["host"]]
})
if "cloud" in data:
cloud = Cloud.objects(name=data["cloud"]).first()
if cloud:
_args["cloud"] = cloud
if self.name == "current_schedule":
_schedule = self.model.current_schedule(**_args)
if _schedule:
return _schedule.to_json()
else:
return json.dumps({"result": ["No results."]})
return self.model.objects(**_args).to_json()
def create_initial_message(real_owner, cloud, cloud_info, ticket, cc):
_cloud_obj = Cloud.objects(name=cloud).first()
template_file = "initial_message"
irc_bot_ip = conf["ircbot_ipaddr"]
irc_bot_port = conf["ircbot_port"]
irc_bot_channel = conf["ircbot_channel"]
cc_users = conf["report_cc"].split(",")
for user in cc:
cc_users.append("%s@%s" % (user, conf["domain"]))
if conf["email_notify"]:
with open(os.path.join(TEMPLATES_PATH, template_file)) as _file:
template = Template(_file.read())
content = template.render(
cloud_info=cloud_info,
wp_wiki=conf["wp_wiki"],
cloud=cloud,
quads_url=conf["quads_url"],
real_owner=real_owner,
ticket=ticket,
foreman_url=conf["foreman_url"],
)
postman = Postman("New QUADS Assignment Allocated", real_owner,
cc_users, content)
postman.send_email()
if conf["irc_notify"]:
try:
with Netcat(irc_bot_ip, irc_bot_port) as nc:
message = "%s QUADS: %s is now active, choo choo! - http://%s/assignments/#%s" % (
irc_bot_channel, cloud_info, conf["wp_wiki"], cloud)
nc.write(bytes(message.encode("utf-8")))
except (TypeError, BrokenPipeError) as ex:
logger.debug(ex)
logger.error("Beep boop netcat can't communicate with your IRC.")
_cloud_obj.update(notified=True)
def main(_args, _loop):
clouds = Cloud.objects(validated=False, provisioned=True, name__ne="cloud01")
for _cloud in clouds:
_schedule_count = Schedule.current_schedule(cloud=_cloud).count()
if _schedule_count and _cloud.wipe:
validator = Validator(_cloud, _args, _loop=_loop)
try:
_loop.run_until_complete(validator.validate_env())
except Exception as ex:
logger.debug(ex)
logger.info("Failed validation for %s" % _cloud.name)
async def main(_loop):
no_extend_label = "CANNOT_EXTEND"
extend_label = "CAN_EXTEND"
try:
jira = Jira(
Config["jira_url"],
loop=_loop,
)
except JiraException as ex:
logger.error(ex)
return 1
tickets = await jira.get_pending_tickets()
for ticket in tickets["issues"]:
ticket_key = ticket.get("key").split("-")[-1]
fields = ticket.get("fields")
if fields:
description = fields.get("description")
try:
cloud_field = description.split("\n")[1]
cloud = cloud_field.split()[-1]
except IndexError:
logger.warning(
f"Could not retrieve cloud name from ticket {ticket_key}")
cloud_obj = Cloud.objects(name=cloud).first()
schedules = Schedule.current_schedule(cloud=cloud_obj)
conflict = False
for schedule in schedules:
end_date = schedule.end + timedelta(weeks=2)
available = Schedule.is_host_available(host=schedule.host.name,
start=schedule.end,
end=end_date)
if not available:
conflict = True
await jira.add_label(ticket_key, no_extend_label)
logger.info(f"{cloud} labeled {no_extend_label}")
break
if not conflict:
await jira.add_label(ticket_key, extend_label)
logger.info(f"{cloud} labeled {extend_label}")
parent = fields.get("parent")
if parent:
p_ticket_key = parent.get("key").split("-")[-1]
watchers = await jira.get_watchers(p_ticket_key)
for watcher in watchers["watchers"]:
await jira.add_watcher(ticket_key, watcher["key"])
return 0
def main() -> None:
"""
Main function
:return: None
"""
cloud = Cloud.objects(name="cloud01").first()
hosts = Host.objects(cloud=cloud, retired=False, broken=False)
for host in hosts:
file_name = f"{host.name}.xml"
file_path = os.path.join(LSHW_OUTPUT_DIR, file_name)
if os.path.exists(file_path):
if os.path.getsize(file_path) < 1:
run_lshw(host.name, file_path)
else:
run_lshw(host.name, file_path)
def verify(cloud):
_cloud_obj = Cloud.objects(name=cloud).first()
logger.info(f"Cloud qinq: {_cloud_obj.qinq}")
if not _cloud_obj:
logger.error(f"Cloud not found.")
return
if args.all:
hosts = Host.objects(cloud=_cloud_obj)
else:
hosts = [Host.objects(cloud=_cloud_obj).first()]
for host in hosts:
if args.all:
logger.info(f"{host.name}:")
if host and host.interfaces:
interfaces = sorted(host.interfaces, key=lambda k: k["name"])
for i, interface in enumerate(interfaces):
ssh_helper = SSHHelper(interface.switch_ip,
Config["junos_username"])
try:
_, vlan_member_out = ssh_helper.run_cmd(
"show configuration vlans | display set | match %s.0" %
interface.switch_port)
vlan_member = vlan_member_out[0].split()[2][4:].strip(",")
except IndexError:
logger.warning(
"Could not determine the previous VLAN member for %s, switch %s, switch port %s "
% (
interface.name,
interface.switch_ip,
interface.switch_port,
))
vlan_member = 0
ssh_helper.disconnect()
logger.info(
f"Interface em{i+1} appears to be a member of VLAN {vlan_member}",
)
else:
logger.error(
f"The cloud has no hosts or the host has no interfaces defined"
)
def report_detailed(_logger, _start, _end):
start = _start.replace(hour=21, minute=59, second=0)
start_defer = start - timedelta(weeks=1)
start_defer_id = date_to_object_id(start_defer)
end = _end.replace(hour=22, minute=1, second=0)
end_id = date_to_object_id(end)
cloud_history = CloudHistory.objects(__raw__={
"_id": {
"$lt": end_id,
"$gt": start_defer_id,
},
}).order_by("-_id")
headers = [
"Owner",
"Ticket",
"Cloud",
"Description",
"Systems",
"Scheduled",
"Duration",
]
_logger.info(f"{headers[0]:<9}| "
f"{headers[1]:>9}| "
f"{headers[2]:>8}| "
f"{headers[3]:>10}| "
f"{headers[4]:>5}| "
f"{headers[5]:>10}| "
f"{headers[6]:>5}| ")
for cloud in cloud_history:
cloud_ref = Cloud.objects(name=cloud.name).first()
schedule = Schedule.objects(
Q(end__lt=end) & Q(start__gt=start)
& Q(cloud=cloud_ref)).order_by("-_id")
if schedule:
delta = schedule[0].end - schedule[0].start
description = cloud.description[:len(headers[3])]
_logger.info(f"{cloud.owner:<9}| "
f"{cloud.ticket:>9}| "
f"{cloud.name:>8}| "
f"{description:>11}| "
f"{schedule.count():>7}| "
f"{str(schedule[0].start)[:10]:>9}| "
f"{delta.days:>8}| ")
def render_vlans(markdown):
lines = []
vlans = Vlan.objects().all()
for vlan in vlans:
cloud_obj = Cloud.objects(vlan=vlan).first()
vlan_id = vlan.vlan_id
ip_range = vlan.ip_range
netmask = vlan.netmask
gateway = vlan.gateway
ip_free = vlan.ip_free
cloud_current_count = Schedule.current_schedule(
cloud=cloud_obj).count()
if cloud_obj and cloud_current_count > 0:
owner = cloud_obj.owner
ticket = cloud_obj.ticket
cloud_name = cloud_obj.name
else:
owner = "nobody"
ticket = ""
cloud_name = ""
columns = [
vlan_id,
ip_range.strip(","),
netmask,
gateway,
ip_free,
owner,
ticket,
cloud_name,
]
lines.append(columns)
for line in sorted(lines, key=lambda _line: _line[1]):
entry = "| %s |\n" % " | ".join([str(col) for col in line])
markdown.write(entry)
def main():
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
future_days = 7
_all_clouds = Cloud.objects()
_active_clouds = [
_cloud for _cloud in _all_clouds
if Schedule.current_schedule(cloud=_cloud).count() > 0
]
_validated_clouds = [
_cloud for _cloud in _active_clouds if _cloud.validated
]
if not os.path.exists(os.path.join(conf["data_dir"], "report")):
Path(os.path.join(conf["data_dir"], "report")).mkdir(parents=True,
exist_ok=True)
for cloud in _validated_clouds:
notification_obj = Notification.objects(cloud=cloud,
ticket=cloud.ticket).first()
current_hosts = Schedule.current_schedule(cloud=cloud)
cloud_info = "%s: %s (%s)" % (
cloud.name,
current_hosts.count(),
cloud.description,
)
if not notification_obj.initial:
logger.info("=============== Initial Message")
loop.run_until_complete(
create_initial_message(
cloud.owner,
cloud.name,
cloud_info,
cloud.ticket,
cloud.ccuser,
))
notification_obj.update(initial=True)
for day in Days:
future = datetime.now() + timedelta(days=day.value)
future_date = "%4d-%.2d-%.2d 22:00" % (
future.year,
future.month,
future.day,
)
future_hosts = Schedule.current_schedule(cloud=cloud,
date=future_date)
diff = set(current_hosts) - set(future_hosts)
if diff and future > current_hosts[0].end:
if not notification_obj[
day.name.lower()] and conf["email_notify"]:
logger.info("=============== Additional Message")
host_list = [schedule.host.name for schedule in diff]
create_message(
cloud,
day.value,
cloud_info,
host_list,
)
kwargs = {day.name.lower(): True}
notification_obj.update(**kwargs)
break
for cloud in _all_clouds:
notification_obj = Notification.objects(cloud=cloud,
ticket=cloud.ticket).first()
if cloud.name != "cloud01" and cloud.owner not in ["quads", None]:
current_hosts = Schedule.current_schedule(cloud=cloud)
cloud_info = "%s: %s (%s)" % (
cloud.name,
current_hosts.count(),
cloud.description,
)
if not notification_obj.pre_initial and conf["email_notify"]:
logger.info("=============== Future Initial Message")
create_future_initial_message(
cloud,
cloud_info,
)
notification_obj.update(pre_initial=True)
for day in range(1, future_days + 1):
if not notification_obj.pre and cloud.validated:
future = datetime.now() + timedelta(days=day)
future_date = "%4d-%.2d-%.2d 22:00" % (
future.year,
future.month,
future.day,
)
future_hosts = Schedule.current_schedule(cloud=cloud,
date=future_date)
if future_hosts.count() > 0:
diff = set(current_hosts) - set(future_hosts)
host_list = [schedule.host.name for schedule in diff]
if diff:
logger.info("=============== Additional Message")
create_future_message(
cloud,
day,
cloud_info,
host_list,
)
notification_obj.update(pre=True)
break
def switch_config(host, old_cloud, new_cloud):
_host_obj = Host.objects(name=host).first()
_old_cloud_obj = Cloud.objects(name=old_cloud).first()
_new_cloud_obj = Cloud.objects(name=new_cloud).first()
if not _host_obj.interfaces:
logger.error("Host has no interfaces defined.")
return False
logger.debug("Connecting to switch on: %s" %
_host_obj.interfaces[0].switch_ip)
switch_ip = None
ssh_helper = None
interfaces = sorted(_host_obj.interfaces, key=lambda k: k["name"])
for i, interface in enumerate(interfaces):
last_nic = i == len(_host_obj.interfaces) - 1
if not switch_ip:
switch_ip = interface.switch_ip
try:
ssh_helper = SSHHelper(switch_ip, Config["junos_username"])
except SSHHelperException:
logger.error(f"Failed to connect to switch: {switch_ip}")
return False
else:
if switch_ip != interface.switch_ip:
ssh_helper.disconnect()
switch_ip = interface.switch_ip
ssh_helper = SSHHelper(switch_ip, Config["junos_username"])
result, old_vlan_out = ssh_helper.run_cmd(
"show configuration interfaces %s" % interface.switch_port)
old_vlan = None
if result and old_vlan_out:
old_vlan = old_vlan_out[0].split(";")[0].split()[1][7:]
if not old_vlan:
if not _new_cloud_obj.vlan and not last_nic:
logger.warning(
"Warning: Could not determine the previous VLAN for %s on %s, switch %s, switchport %s"
% (
host,
interface.name,
interface.switch_ip,
interface.switch_port,
))
old_vlan = get_vlan(_old_cloud_obj, i)
new_vlan = get_vlan(_new_cloud_obj, i)
if _new_cloud_obj.vlan and last_nic:
if int(old_vlan) != int(_new_cloud_obj.vlan.vlan_id):
logger.info("Setting last interface to public vlan %s." %
new_vlan)
juniper = Juniper(
interface.switch_ip,
interface.switch_port,
old_vlan,
_new_cloud_obj.vlan.vlan_id,
)
success = juniper.convert_port_public()
if success:
logger.info("Successfully updated switch settings.")
else:
logger.error(
"There was something wrong updating switch for %s:%s" %
(host, interface.name))
return False
else:
if int(old_vlan) != int(new_vlan):
juniper = Juniper(interface.switch_ip, interface.switch_port,
old_vlan, new_vlan)
success = juniper.set_port()
if success:
logger.info("Successfully updated switch settings.")
else:
logger.error(
"There was something wrong updating switch for %s:%s" %
(host, interface.name))
return False
if ssh_helper:
ssh_helper.disconnect()
return True
async def move_and_rebuild(host,
new_cloud,
semaphore,
rebuild=False,
loop=None):
build_start = datetime.now()
logger.debug("Moving and rebuilding host: %s" % host)
untouchable_hosts = Config["untouchable_hosts"]
logger.debug("Untouchable hosts: %s" % untouchable_hosts)
_host_obj = Host.objects(name=host).first()
if host in untouchable_hosts:
logger.error("No way...")
return False
_target_cloud = Cloud.objects(name=new_cloud).first()
ipmi_new_pass = (f"{Config['infra_location']}@{_target_cloud.ticket}"
if _target_cloud.ticket else Config["ipmi_password"])
ipmi_set_pass = [
"user",
"set",
"password",
str(Config["ipmi_cloud_username_id"]),
ipmi_new_pass,
]
new_semaphore = asyncio.Semaphore(20)
await execute_ipmi(host, arguments=ipmi_set_pass, semaphore=new_semaphore)
ipmi_set_operator = [
"user", "priv",
str(Config["ipmi_cloud_username_id"]), "0x4"
]
await execute_ipmi(host,
arguments=ipmi_set_operator,
semaphore=new_semaphore)
badfish = None
if rebuild and _target_cloud.name != _host_obj.default_cloud.name:
if Config.pdu_management:
# TODO: pdu management
pass
try:
badfish = await badfish_factory(
"mgmt-%s" % host,
Config["ipmi_username"],
Config["ipmi_password"],
propagate=True,
)
except BadfishException:
logger.error(
f"Could not initialize Badfish. Verify ipmi credentials for mgmt-{host}."
)
return False
if is_supported(host):
try:
interfaces_path = os.path.join(
os.path.dirname(__file__),
"../../conf/idrac_interfaces.yml")
await badfish.change_boot("director", interfaces_path)
# wait 10 minutes for the boot order job to complete
await asyncio.sleep(600)
except BadfishException:
logger.error(
f"Could not set boot order via Badfish for mgmt-{host}.")
return False
try:
await badfish.set_power_state("on")
except BadfishException:
logger.error(f"Failed to power on {host}")
return False
foreman_results = []
params = [
{
"name": "operatingsystems",
"value": Config["foreman_default_os"],
"identifier": "title",
},
{
"name": "ptables",
"value": Config["foreman_default_ptable"]
},
{
"name": "media",
"value": Config["foreman_default_medium"]
},
]
foreman = Foreman(
Config["foreman_api_url"],
Config["foreman_username"],
Config["foreman_password"],
semaphore=semaphore,
loop=loop,
)
set_result = await foreman.set_host_parameter(host, "overcloud",
"true")
foreman_results.append(set_result)
put_result = await foreman.put_parameter(host, "build", 1)
foreman_results.append(put_result)
put_param_result = await foreman.put_parameters_by_name(host, params)
foreman_results.append(put_param_result)
owner_id = await foreman.get_user_id(new_cloud)
host_id = await foreman.get_host_id(host)
put_result = await foreman.put_element("hosts", host_id, "owner_id",
owner_id)
foreman_results.append(put_result)
for result in foreman_results:
if isinstance(result, Exception) or not result:
logger.error(
"There was something wrong setting Foreman host parameters."
)
return False
if is_supported(host):
try:
await badfish.boot_to_type(
"foreman",
os.path.join(os.path.dirname(__file__),
"../../conf/idrac_interfaces.yml"),
)
await badfish.reboot_server(graceful=False)
except BadfishException:
logger.error(f"Error setting PXE boot via Badfish on {host}.")
await badfish.reboot_server(graceful=False)
return False
else:
try:
asyncio.run_coroutine_threadsafe(
badfish.unmount_virtual_media(),
loop,
)
except BadfishException:
logger.warning(
f"Could not unmount virtual media for mgmt-{host}.")
try:
ipmi_pxe_persistent = [
"chassis",
"bootdev",
"pxe",
"options=persistent",
]
await execute_ipmi(host,
arguments=ipmi_pxe_persistent,
semaphore=new_semaphore)
await ipmi_reset(host, new_semaphore)
except Exception as ex:
logger.debug(ex)
logger.error(
f"There was something wrong setting PXE flag or resetting IPMI on {host}."
)
if _target_cloud.name == _host_obj.default_cloud.name:
if not badfish:
try:
badfish = await badfish_factory(
"mgmt-%s" % host,
Config["ipmi_username"],
Config["ipmi_password"],
propagate=True,
)
except BadfishException:
logger.error(
f"Could not initialize Badfish. Verify ipmi credentials for mgmt-{host}."
)
return False
await badfish.set_power_state("off")
source_cloud_schedule = Schedule.current_schedule(
cloud=_host_obj.cloud.name)
if not source_cloud_schedule:
_old_cloud_obj = Cloud.objects(name=_host_obj.cloud.name).first()
_old_cloud_obj.update(vlan=None)
schedule = Schedule.current_schedule(cloud=_target_cloud,
host=_host_obj).first()
if schedule:
schedule.update(build_start=build_start, build_end=datetime.now())
schedule.save()
logger.debug("Updating host: %s")
_host_obj.update(cloud=_target_cloud,
build=False,
last_build=datetime.now(),
validated=False)
return True
ssh_helper.disconnect()
return True
def validate_env(_cloud):
if not post_system_test(_cloud):
if env_allocation_time_exceeded(_cloud):
notify_failure(_cloud)
return
if not post_network_test(_cloud):
if env_allocation_time_exceeded(_cloud):
notify_failure(_cloud)
return
# TODO: gather ansible-cmdb facts
# TODO: quads dell config report
notify_success(_cloud)
_cloud.update(validated=True)
return
if __name__ == "__main__":
clouds = Cloud.objects(released=True, validated=False, name__ne="cloud01")
for cloud in clouds:
validate_env(cloud)
def make_env_json(filename):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
foreman = Foreman(
conf["foreman_api_url"],
conf["foreman_username"],
conf["foreman_password"],
loop=loop,
)
cloud_list = Cloud.objects()
if not os.path.exists(conf["json_web_path"]):
os.makedirs(conf["json_web_path"])
now = time.time()
old_jsons = [
file for file in os.listdir(conf["json_web_path"]) if ":" in file
]
for file in old_jsons:
if (os.stat(os.path.join(conf["json_web_path"], file)).st_mtime <
now - conf["json_retention_days"] * 86400):
os.remove(os.path.join(conf["json_web_path"], file))
for cloud in cloud_list:
host_list = Host.objects(cloud=cloud).order_by("name")
foreman_password = conf["ipmi_password"]
if cloud.ticket:
foreman_password = f"{conf['infra_location']}@{cloud.ticket}"
data = defaultdict(list)
for host in host_list:
if conf["foreman_unavailable"]:
overcloud = {"result": "true"}
else:
overcloud = loop.run_until_complete(
foreman.get_host_param(host.name, "overcloud"))
if not overcloud:
overcloud = {"result": "true"}
if type(overcloud["result"]) != bool:
try:
_overcloud_result = strtobool(overcloud["result"])
except ValueError:
print(
f"WARN: {host.name} overcloud value is not set correctly."
)
_overcloud_result = 1
else:
_overcloud_result = overcloud["result"]
if "result" in overcloud and _overcloud_result:
mac = []
if filename == "instackenv":
for interface in host.interfaces:
if interface.pxe_boot:
mac.append(interface.mac_address)
if filename == "ocpinventory":
mac = [
interface.mac_address for interface in host.interfaces
]
data["nodes"].append({
"pm_password": foreman_password,
"pm_type": "pxe_ipmitool",
"mac": mac,
"cpu": "2",
"memory": "1024",
"disk": "20",
"arch": "x86_64",
"pm_user": conf["ipmi_cloud_username"],
"pm_addr": "mgmt-%s" % host.name,
})
content = json.dumps(data, indent=4, sort_keys=True)
if not os.path.exists(conf["json_web_path"]):
pathlib.Path(conf["json_web_path"]).mkdir(parents=True,
exist_ok=True)
now = datetime.now()
new_json_file = os.path.join(
conf["json_web_path"],
"%s_%s.json_%s" %
(cloud.name, filename, now.strftime("%Y-%m-%d_%H:%M:%S")),
)
json_file = os.path.join(conf["json_web_path"],
"%s_%s.json" % (cloud.name, filename))
with open(new_json_file, "w+") as _json_file:
_json_file.seek(0)
_json_file.write(content)
os.chmod(new_json_file, 0o644)
copyfile(new_json_file, json_file)
async def move_and_rebuild(host,
new_cloud,
semaphore,
rebuild=False,
loop=None):
build_start = datetime.now()
logger.debug("Moving and rebuilding host: %s" % host)
untouchable_hosts = conf["untouchable_hosts"]
logger.debug("Untouchable hosts: %s" % untouchable_hosts)
_host_obj = Host.objects(name=host).first()
if host in untouchable_hosts:
logger.error("No way...")
return False
_new_cloud_obj = Cloud.objects(name=new_cloud).first()
ipmi_new_pass = (f"{conf['infra_location']}@{_new_cloud_obj.ticket}"
if _new_cloud_obj.ticket else conf["ipmi_password"])
ipmi_set_pass = [
"user",
"set",
"password",
str(conf["ipmi_cloud_username_id"]),
ipmi_new_pass,
]
new_semaphore = asyncio.Semaphore(20)
await execute_ipmi(host, arguments=ipmi_set_pass, semaphore=new_semaphore)
ipmi_set_operator = [
"user", "priv",
str(conf["ipmi_cloud_username_id"]), "0x4"
]
await execute_ipmi(host,
arguments=ipmi_set_operator,
semaphore=new_semaphore)
if rebuild and _new_cloud_obj.name != _host_obj.default_cloud.name:
if "pdu_management" in conf and conf["pdu_management"]:
# TODO: pdu management
pass
if is_supported(host):
try:
badfish = await badfish_factory(
"mgmt-%s" % host,
conf["ipmi_username"],
conf["ipmi_password"],
propagate=True,
)
except BadfishException:
logger.error(
f"Could not initialize Badfish. Verify ipmi credentials for mgmt-{host}."
)
return False
try:
changed_boot_order = asyncio.run_coroutine_threadsafe(
badfish.change_boot(
"director",
os.path.join(os.path.dirname(__file__),
"../../conf/idrac_interfaces.yml"),
),
loop,
)
if changed_boot_order:
await badfish.reboot_server(graceful=False)
except BadfishException:
logger.error(
f"Could not set boot order via Badfish for mgmt-{host}.")
return False
foreman_results = []
params = [
{
"name": "operatingsystems",
"value": conf["foreman_default_os"],
"identifier": "title",
},
{
"name": "ptables",
"value": conf["foreman_default_ptable"]
},
{
"name": "media",
"value": conf["foreman_default_medium"]
},
]
foreman = Foreman(
conf["foreman_api_url"],
conf["foreman_username"],
conf["foreman_password"],
semaphore=semaphore,
loop=loop,
)
set_result = await foreman.set_host_parameter(host, "overcloud",
"true")
foreman_results.append(set_result)
put_result = await foreman.put_parameter(host, "build", 1)
foreman_results.append(put_result)
put_param_result = await foreman.put_parameters_by_name(host, params)
foreman_results.append(put_param_result)
owner_id = await foreman.get_user_id(new_cloud)
host_id = await foreman.get_host_id(host)
put_result = await foreman.put_element("hosts", host_id, "owner_id",
owner_id)
foreman_results.append(put_result)
for result in foreman_results:
if isinstance(result, Exception) or not result:
logger.error(
"There was something wrong setting Foreman host parameters."
)
return False
healthy = False
for i in range(RETRIES):
nc = Netcat(_host_obj.name)
if nc.health_check():
healthy = True
nc.close()
break
nc.close()
if not healthy:
logger.error("Failed to recover host after changing boot order.")
return False
if is_supported(host):
try:
await badfish.boot_to_type(
"foreman",
os.path.join(os.path.dirname(__file__),
"../../conf/idrac_interfaces.yml"),
)
await badfish.reboot_server(graceful=False)
except BadfishException:
logger.error(f"Error setting PXE boot via Badfish on {host}.")
await badfish.reboot_server(graceful=False)
return False
else:
try:
ipmi_pxe_persistent = [
"chassis",
"bootdev",
"pxe",
"options",
"=",
"persistent",
]
await execute_ipmi(host,
arguments=ipmi_pxe_persistent,
semaphore=new_semaphore)
await ipmi_reset(host, new_semaphore)
except Exception as ex:
logger.debug(ex)
logger.error(
f"There was something wrong setting PXE flag or resetting IPMI on {host}."
)
schedule = Schedule.current_schedule(cloud=_new_cloud_obj,
host=_host_obj).first()
if schedule:
schedule.update(build_start=build_start, build_end=datetime.now())
schedule.save()
logger.debug("Updating host: %s")
_host_obj.update(cloud=_new_cloud_obj,
build=False,
last_build=datetime.now(),
validated=False)
return True
def GET(self, **data):
args = {}
_cloud = None
_host = None
if "cloudonly" in data:
_cloud = Cloud.objects(cloud=data["cloudonly"])
if not _cloud:
cherrypy.response.status = "404 Not Found"
return json.dumps(
{"result": "Cloud %s Not Found" % data["cloudonly"]})
else:
return _cloud.to_json()
if self.name == "host":
if "id" in data:
_host = Host.objects(id=data["id"]).first()
elif "name" in data:
_host = Host.objects(name=data["name"]).first()
elif "cloud" in data:
_cloud = Cloud.objects(name=data["cloud"]).first()
_host = Host.objects(cloud=_cloud)
else:
_host = Host.objects()
if not _host:
return json.dumps({"result": ["Nothing to do."]})
return _host.to_json()
if self.name == "ccuser":
_clouds = self.model.objects().all()
clouds_summary = []
for cloud in _clouds:
count = Schedule.current_schedule(cloud=cloud).count()
clouds_summary.append({
"name": cloud.name,
"count": count,
"description": cloud.description,
"owner": cloud.owner,
"ticket": cloud.ticket,
"ccuser": cloud.ccuser,
"provisioned": cloud.provisioned,
})
return json.dumps(clouds_summary)
if self.name == "cloud":
if "id" in data:
_cloud = Cloud.objects(id=data["id"]).first()
elif "name" in data:
_cloud = Cloud.objects(name=data["name"]).first()
elif "owner" in data:
_cloud = Cloud.to_json(owner=data["owner"]).first()
if _cloud:
return _cloud.to_json()
if self.name == "available":
_start = _end = datetime.datetime.now()
if "start" in data:
_start = datetime.datetime.strptime(data["start"],
"%Y-%m-%dT%H:%M:%S")
if "end" in data:
_end = datetime.datetime.strptime(data["end"],
"%Y-%m-%dT%H:%M:%S")
available = []
all_hosts = Host.objects().all()
for host in all_hosts:
if Schedule.is_host_available(host=host["name"],
start=_start,
end=_end):
available.append(host.name)
return json.dumps(available)
if self.name == "summary":
_clouds = Cloud.objects().all()
clouds_summary = []
total_count = 0
for cloud in _clouds:
if cloud.name == "cloud01":
count = Host.objects(cloud=cloud,
retired=False,
broken=False).count()
else:
date = datetime.datetime.now()
if "date" in data:
date = datetime.datetime.strptime(
data["date"], "%Y-%m-%dT%H:%M:%S")
count = self.model.current_schedule(cloud=cloud,
date=date).count()
total_count += count
clouds_summary.append({
"name": cloud.name,
"count": count,
"description": cloud.description,
"owner": cloud.owner,
"ticket": cloud.ticket,
"ccuser": cloud.ccuser,
"provisioned": cloud.provisioned,
"validated": cloud.validated,
})
if "date" in data:
host_count = Host.objects(retired=False, broken=False).count()
for cloud in clouds_summary:
if cloud["name"] == "cloud01":
cloud["count"] = host_count - total_count
return json.dumps(clouds_summary)
if self.name == "qinq":
_clouds = Cloud.objects().all()
clouds_qinq = []
for cloud in _clouds:
_type = "Isolated"
if cloud.qinq == 1:
_type = "Combined"
qinq_value = f"{cloud.qinq} ({_type})"
clouds_qinq.append({"name": cloud.name, "qinq": qinq_value})
return json.dumps(clouds_qinq)
if self.name == "broken":
_hosts = self.model.objects(broken=True)
broken = []
for host in _hosts:
broken.append(host.name)
return json.dumps(broken)
if self.name == "retired":
hosts = [host.name for host in self.model.objects(retired=True)]
return json.dumps(hosts)
objs = self.model.objects(**args)
if objs:
return objs.to_json()
else:
return json.dumps({"result": ["No results."]})
return
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Validate Quads assignments")
parser.add_argument(
"--debug",
action="store_true",
default=False,
help="Show debugging information.",
)
args = parser.parse_args()
level = logging.INFO
if args.debug:
level = logging.DEBUG
logging.basicConfig(level=level, format="%(message)s")
clouds = Cloud.objects(validated=False, provisioned=True, name__ne="cloud01")
for _cloud in clouds:
_schedule_count = Schedule.current_schedule(cloud=_cloud).count()
if _schedule_count and _cloud.wipe:
validator = Validator(_cloud)
try:
validator.validate_env()
except Exception as ex:
logger.debug(ex)
logger.info("Failed validation for %s" % _cloud.name)
def verify(_cloud_name, change=False):
_cloud_obj = Cloud.objects(name=_cloud_name).first()
hosts = Host.objects(cloud=_cloud_obj)
for _host_obj in hosts:
logger.info(f"Host: {_host_obj.name}")
if _host_obj.interfaces:
interfaces = sorted(_host_obj.interfaces, key=lambda k: k["name"])
for i, interface in enumerate(interfaces):
ssh_helper = SSHHelper(interface.ip_address,
conf["junos_username"])
last_nic = i == len(_host_obj.interfaces) - 1
vlan = get_vlan(_cloud_obj, i, last_nic)
try:
old_vlan_out = ssh_helper.run_cmd(
"show configuration interfaces %s" %
interface.switch_port)
old_vlan = old_vlan_out[0].split(";")[0].split()[1]
if old_vlan.startswith("QinQ"):
old_vlan = old_vlan[7:]
except IndexError:
old_vlan = None
try:
vlan_member_out = ssh_helper.run_cmd(
"show configuration vlans | display set | match %s.0" %
interface.switch_port)
vlan_member = vlan_member_out[0].split()[2][4:].strip(",")
except IndexError:
if not _cloud_obj.vlan and not last_nic:
logger.warning(
"Could not determine the previous VLAN member for %s, switch %s, switch port %s "
% (
interface.name,
interface.ip_address,
interface.switch_port,
))
vlan_member = None
ssh_helper.disconnect()
if not old_vlan:
if not _cloud_obj.vlan and not last_nic:
logger.warning(
"Could not determine the previous VLAN for %s, switch %s, switchport %s"
% (
interface.name,
interface.ip_address,
interface.switch_port,
))
old_vlan = get_vlan(_cloud_obj, i, last_nic)
if int(old_vlan) != int(vlan):
logger.warning("Interface %s not using QinQ_vl%s",
interface.switch_port, vlan)
if _cloud_obj.vlan and i == len(_host_obj.interfaces) - 1:
vlan = _cloud_obj.vlan.vlan_id
if not vlan_member or int(vlan_member) != int(vlan):
if not _cloud_obj.vlan and not last_nic:
logger.warning(
"Interface %s appears to be a member of VLAN %s, should be %s",
interface.switch_port,
vlan_member,
vlan,
)
if change:
if _cloud_obj.vlan and last_nic:
if int(_cloud_obj.vlan.vlan_id) != int(old_vlan):
logger.info(
f"Change requested for {interface.name}")
logger.info(
"Setting last interface to public vlan %s."
% _cloud_obj.vlan.vlan_id)
success = juniper_convert_port_public(
interface.ip_address,
interface.switch_port,
old_vlan,
vlan,
)
else:
logger.info(
f"No changes required for {interface.name}"
)
continue
else:
logger.info(
f"Change requested for {interface.name}")
success = juniper_set_port(
interface.ip_address,
interface.switch_port,
vlan_member,
vlan,
)
if success:
logger.info(
"Successfully updated switch settings.")
else:
logger.error(
f"There was something wrong updating switch for {interface.name}"
)
def main():
if conf["openstack_management"]:
foreman = Foreman(conf["foreman_api_url"], conf["foreman_username"],
conf["foreman_password"])
cloud_list = Cloud.objects()
if not os.path.exists(conf["json_web_path"]):
os.makedirs(conf["json_web_path"])
now = time.time()
old_jsons = [
file for file in os.listdir(conf["json_web_path"]) if ":" in file
]
for file in old_jsons:
if os.stat(os.path.join(conf["json_web_path"], file)
).st_mtime < now - conf["json_retention_days"] * 86400:
os.remove(os.path.join(conf["json_web_path"], file))
for cloud in cloud_list:
host_list = Host.objects(cloud=cloud).order_by("name")
foreman_password = conf["ipmi_password"]
if cloud.ticket:
foreman_password = cloud.ticket
json_data = defaultdict(list)
for host in host_list[1:]:
if conf["foreman_unavailable"]:
overcloud = {"result": "true"}
else:
overcloud = foreman.get_host_param(host.name, "overcloud")
if not overcloud:
overcloud = {"result": "true"}
if "result" in overcloud and strtobool(overcloud["result"]):
mac = "00:00:00:00:00:00"
if len(host.interfaces) > 1:
mac = host.interfaces[1].mac_address
json_data['nodes'].append({
'pm_password':
foreman_password,
'pm_type':
"pxe_ipmitool",
'mac': [mac],
'cpu':
"2",
'memory':
"1024",
'disk':
"20",
'arch':
"x86_64",
'pm_user':
conf["ipmi_cloud_username"],
'pm_addr':
"mgmt-%s" % host.name
})
content = json.dumps(json_data, indent=4, sort_keys=True)
if not os.path.exists(conf["json_web_path"]):
pathlib.Path(conf["json_web_path"]).mkdir(parents=True,
exist_ok=True)
now = datetime.now()
new_json_file = os.path.join(
conf["json_web_path"], "%s_instackenv.json_%s" %
(cloud.name, now.strftime("%Y-%m-%d_%H:%M:%S")))
json_file = os.path.join(conf["json_web_path"],
"%s_instackenv.json" % cloud.name)
with open(new_json_file, "w+") as _json_file:
_json_file.seek(0)
_json_file.write(content)
os.chmod(new_json_file, 0o644)
copyfile(new_json_file, json_file)
def main():
days = [1, 3, 5, 7]
future_days = 7
_all_clouds = Cloud.objects()
_active_clouds = [
_cloud for _cloud in _all_clouds
if Schedule.current_schedule(cloud=_cloud).count() > 0
]
_validated_clouds = [
_cloud for _cloud in _active_clouds if _cloud.validated
]
if not os.path.exists(os.path.join(conf["data_dir"], "report")):
Path(os.path.join(conf["data_dir"], "report")).mkdir(parents=True,
exist_ok=True)
for cloud in _validated_clouds:
current_hosts = Schedule.current_schedule(cloud=cloud)
cloud_info = "%s: %s (%s)" % (cloud.name, current_hosts.count(),
cloud.description)
if not cloud["notified"]:
logger.info('=============== Initial Message')
create_initial_message(
cloud.owner,
cloud.name,
cloud_info,
cloud.ticket,
cloud.ccuser,
)
for day in days:
future = datetime.now() + timedelta(days=day)
future_date = "%4d-%.2d-%.2d 22:00" % (future.year, future.month,
future.day)
future_hosts = Schedule.current_schedule(cloud=cloud,
date=future_date)
diff = set(current_hosts) - set(future_hosts)
if diff and future < current_hosts[0].end:
report_file = "%s-%s-%s-%s" % (cloud.name, cloud.owner, day,
cloud.ticket)
report_path = os.path.join(conf["data_dir"], "report",
report_file)
if not os.path.exists(report_path) and conf["email_notify"]:
logger.info('=============== Additional Message')
host_list = [schedule.host.name for schedule in diff]
create_message(
cloud.owner,
day,
cloud.name,
cloud_info,
cloud.ccuser,
host_list,
report_path,
)
break
for cloud in _all_clouds:
if cloud.name != "cloud01" and cloud.owner not in ["quads", None]:
current_hosts = Schedule.current_schedule(cloud=cloud)
cloud_info = "%s: %s (%s)" % (cloud.name, current_hosts.count(),
cloud.description)
report_pre_ini_file = "%s-%s-pre-initial-%s" % (
cloud.name, cloud.owner, cloud.ticket)
report_pre_ini_path = os.path.join(conf["data_dir"], "report",
report_pre_ini_file)
if not os.path.exists(
report_pre_ini_path) and conf["email_notify"]:
logger.info('=============== Future Initial Message')
create_future_initial_message(
cloud.owner,
cloud_info,
cloud.ccuser,
report_pre_ini_path,
)
report_pre_file = "%s-%s-pre-%s" % (cloud.name, cloud.owner,
cloud.ticket)
report_pre_path = os.path.join(conf["data_dir"], "report",
report_pre_file)
if not os.path.exists(report_pre_path) and cloud.validated:
future = datetime.now() + timedelta(days=future_days)
future_date = "%4d-%.2d-%.2d 22:00" % (
future.year, future.month, future.day)
future_hosts = Schedule.current_schedule(cloud=cloud,
date=future_date)
diff = set(current_hosts) - set(future_hosts)
host_list = [schedule.host.name for schedule in diff]
if diff:
logger.info('=============== Additional Message')
create_future_message(
cloud.owner,
future_days,
cloud.name,
cloud_info,
cloud.ccuser,
host_list,
report_pre_path,
)
def verify(_cloud_name, change=False):
_cloud_obj = Cloud.objects(name=_cloud_name).first()
quads = Api(API_URL)
hosts = quads.get_cloud_hosts(_cloud_name)
for _host in hosts:
_host_obj = Host.objects(name=_host["name"]).first()
if _host_obj.interfaces:
_public_vlan_obj = Vlan.objects(cloud=_cloud_obj).first()
for i, interface in enumerate(_host_obj.interfaces):
ssh_helper = SSHHelper(interface.ip_address,
conf["junos_username"])
vlan = get_vlan(_cloud_obj, i)
try:
old_vlan_out = ssh_helper.run_cmd(
"show configuration interfaces %s" %
interface.switch_port)
old_vlan = old_vlan_out[0].split(";")[0].split()[1][7:]
except IndexError:
old_vlan = None
try:
vlan_member_out = ssh_helper.run_cmd(
"show vlans interface %s.0" % interface.switch_port)
vlan_member = vlan_member_out[1].split()[2][4:].strip(",")
except IndexError:
vlan_member = None
ssh_helper.disconnect()
if not old_vlan:
logger.warning(
"Could not determine the previous VLAN for %s on %s, switch %s, switchport %s"
% (_host["name"], interface.name, interface.ip_address,
interface.switch_port))
old_vlan = get_vlan(_cloud_obj, i)
if not vlan_member:
logger.warning(
"Could not determine the previous VLAN member for %s on %s, switch %s, switchport %s"
% (_host["name"], interface.name, interface.ip_address,
interface.switch_port))
vlan_member = old_vlan
if int(old_vlan) != int(vlan):
logger.warning("interface %s not using QinQ_vl%s",
interface.switch_port, vlan)
if _public_vlan_obj and i == len(_host_obj.interfaces) - 1:
vlan = _public_vlan_obj.vlan_id
if int(vlan_member) != int(vlan):
logger.warning(
"interface %s appears to be a member of VLAN %s, should be %s",
interface.switch_port, vlan_member, vlan)
if change:
logger.info('=== INFO: change requested')
if _public_vlan_obj and i == len(
_host_obj.interfaces) - 1:
logger.info(
"Setting last interface to public vlan %s." %
_public_vlan_obj.vlan_id)
success = juniper_convert_port_public(
interface.ip_address, interface.switch_port,
str(old_vlan), str(vlan))
else:
success = juniper_set_port(interface.ip_address,
interface.switch_port,
str(vlan_member),
str(vlan))
if success:
logger.info(
"Successfully updated switch settings.")
else:
logger.error(
"There was something wrong updating switch for %s:%s"
% (_host.name, interface.name))
def move_and_rebuild(host, old_cloud, new_cloud, rebuild=False):
logger.debug("Moving and rebuilding host: %s" % host)
untouchable_hosts = conf["untouchable_hosts"]
logger.debug("Untouchable hosts: %s" % untouchable_hosts)
_host_obj = Host.objects(name=host).first()
if not _host_obj.interfaces:
logger.error("Host has no interfaces defined.")
return False
if host in untouchable_hosts:
logger.error("No way...")
return False
_old_cloud_obj = Cloud.objects(name=old_cloud).first()
_new_cloud_obj = Cloud.objects(name=new_cloud).first()
logger.debug("Connecting to switch on: %s" %
_host_obj.interfaces[0].ip_address)
_public_vlan_obj = Vlan.objects(cloud=_new_cloud_obj).first()
for i, interface in enumerate(_host_obj.interfaces):
ssh_helper = SSHHelper(interface.ip_address, conf["junos_username"])
old_vlan_out = ssh_helper.run_cmd("show configuration interfaces %s" %
interface.switch_port)
old_vlan = old_vlan_out[0].split(";")[0].split()[1][7:]
if not old_vlan:
logger.warning(
"Warning: Could not determine the previous VLAN for %s on %s, switch %s, switchport %s"
% (host, interface.name, interface.ip_address,
interface.switch_port))
old_vlan = get_vlan(_old_cloud_obj, i)
new_vlan = get_vlan(_new_cloud_obj, i)
if _public_vlan_obj and i == len(_host_obj.interfaces) - 1:
logger.info("Setting last interface to public vlan %s." % new_vlan)
if int(old_vlan) != int(_public_vlan_obj.vlan_id):
success = juniper_convert_port_public(
interface.ip_address, interface.switch_port, str(old_vlan),
str(_public_vlan_obj.vlan_id))
if success:
logger.info("Successfully updated switch settings.")
else:
logger.error(
"There was something wrong updating switch for %s:%s" %
(host, interface.name))
return False
else:
if int(old_vlan) != int(new_vlan):
success = juniper_set_port(interface.ip_address,
interface.switch_port,
str(old_vlan), str(new_vlan))
if success:
logger.info("Successfully update switch settings.")
else:
logger.error(
"There was something wrong updating switch for %s:%s" %
(host, interface.name))
return False
if i == len(_host_obj.interfaces) - 1:
_old_vlan_obj = Vlan.objects(cloud=_old_cloud_obj).first()
if _old_vlan_obj:
_old_vlan_obj.update(cloud=None)
ssh_helper.disconnect()
ipmi_new_pass = _new_cloud_obj.ticket if _new_cloud_obj.ticket else conf[
"$ipmi_password"]
foreman = Foreman(
conf["foreman_api_url"],
conf["foreman_username"],
conf["foreman_password"],
)
foreman.remove_role(_old_cloud_obj.name, _host_obj.name)
foreman.add_role(_new_cloud_obj.name, _host_obj.name)
foreman.update_user_password(_new_cloud_obj.name, ipmi_new_pass)
ipmi_set_pass = [
"/usr/bin/ipmitool", "-I", "lanplus", "-H",
"mgmt-%s" % host, "-U", conf["ipmi_username"], "-P",
conf["ipmi_password"], "user", "set", "password",
str(conf["ipmi_cloud_username_id"]), ipmi_new_pass
]
logger.debug("ipmi_set_pass: %s" % ipmi_set_pass)
subprocess.call(ipmi_set_pass)
ipmi_set_operator = [
"/usr/bin/ipmitool", "-I", "lanplus", "-H",
"mgmt-%s" % host, "-U", conf["ipmi_username"], "-P",
conf["ipmi_password"], "user", "priv",
str(conf["ipmi_cloud_username_id"]), "0x4"
]
logger.debug("ipmi_set_operator: %s" % ipmi_set_operator)
subprocess.call(ipmi_set_operator)
if rebuild and _new_cloud_obj.name != "cloud01":
badfish = Badfish("mgmt-%s" % host, conf["ipmi_username"],
conf["ipmi_password"])
if "pdu_management" in conf and conf["pdu_management"]:
# TODO: pdu management
pass
if is_supermicro(host):
ipmi_pxe_persistent = [
"/usr/bin/ipmitool", "-I", "lanplus", "-H",
"mgmt-%s" % host, "-U", conf["ipmi_username"], "-P",
conf["ipmi_password"], "chassis", "bootdev", "pxe", "options",
"=", "persistent"
]
logger.debug("ipmi_pxe_persistent: %s" % ipmi_pxe_persistent)
subprocess.call(ipmi_pxe_persistent)
if is_supported(host):
try:
badfish.change_boot(
"director",
os.path.join(os.path.dirname(__file__),
"../../conf/idrac_interfaces.yml"))
except SystemExit as ex:
logger.debug(ex)
logger.error("Could not set boot order via Badfish.")
return False
foreman_success = foreman.remove_extraneous_interfaces(host)
foreman_success = foreman.set_host_parameter(
host, "rhel73", "false") and foreman_success
foreman_success = foreman.set_host_parameter(
host, "rhel75", "false") and foreman_success
foreman_success = foreman.set_host_parameter(
host, "overcloud", "true") and foreman_success
foreman_success = foreman.put_parameter(host, "build",
1) and foreman_success
foreman_success = foreman.put_parameter_by_name(
host, "operatingsystems", conf["foreman_default_os"],
"title") and foreman_success
foreman_success = foreman.put_parameter_by_name(
host, "ptables",
conf["foreman_default_ptable"]) and foreman_success
foreman_success = foreman.put_parameter_by_name(
host, "media", conf["foreman_default_medium"]) and foreman_success
if not foreman_success:
logger.error(
"There was something wrong setting Foreman host parameters.")
try:
badfish.set_next_boot_pxe()
badfish.reboot_server()
except SystemExit:
if is_supermicro(host):
logger.warning("Badfish not yet supported on Supermicro: %s." %
host)
else:
logger.exception(
"There was something wrong setting next PXE boot via Badfish."
)
return False
logger.debug("Updating host: %s")
_host_obj.update(cloud=_new_cloud_obj,
build=False,
last_build=datetime.now())
else:
logger.debug("Updating host: %s")
_host_obj.update(cloud=_new_cloud_obj,
build=False,
last_build=datetime.now())
ssh_helper.disconnect()
return True
def validate_env(_cloud):
if not post_system_test(_cloud):
if env_allocation_time_exceeded(_cloud):
notify_failure(_cloud)
return
if not post_network_test(_cloud):
if env_allocation_time_exceeded(_cloud):
notify_failure(_cloud)
return
# TODO: gather ansible-cmdb facts
# TODO: quads dell config report
notify_success(_cloud)
_cloud.update(validated=True)
return
if __name__ == "__main__":
clouds = Cloud.objects(released=True, validated=False)
for cloud in clouds:
validate_env(cloud)
def verify(_cloud_name, _host_name, change=False):
if not _cloud_name and not _host_name:
logger.warning(
f"At least one of --cloud or --host should be specified.")
return
_cloud_obj = None
if _cloud_name:
_cloud_obj = Cloud.objects(name=_cloud_name).first()
if _host_name:
hosts = Host.objects(name=_host_name, retired=False)
else:
hosts = Host.objects(cloud=_cloud_obj, retired=False)
first_host = hosts.first()
if not _cloud_obj:
_cloud_obj = first_host.cloud
if _cloud_obj != first_host.cloud:
logger.warning(f"Both --cloud and --host have been specified.")
logger.warning(f"Host: {first_host.name}")
logger.warning(f"Cloud: {_cloud_obj.name}")
logger.warning(
f"However, {first_host.name} is a member of {first_host.cloud.name}"
)
logger.warning(f"!!!!! Be certain this is what you want to do. !!!!!")
for _host_obj in hosts:
logger.info(f"Host: {_host_obj.name}")
if _host_obj.interfaces:
interfaces = sorted(_host_obj.interfaces, key=lambda k: k["name"])
for i, interface in enumerate(interfaces):
ssh_helper = SSHHelper(interface.switch_ip,
Config["junos_username"])
last_nic = i == len(_host_obj.interfaces) - 1
vlan = get_vlan(_cloud_obj, i, last_nic)
try:
_, old_vlan_out = ssh_helper.run_cmd(
"show configuration interfaces %s" %
interface.switch_port)
old_vlan = old_vlan_out[0].split(";")[0].split()[1]
if old_vlan.startswith("QinQ"):
old_vlan = old_vlan[7:]
except IndexError:
old_vlan = 0
try:
_, vlan_member_out = ssh_helper.run_cmd(
"show configuration vlans | display set | match %s.0" %
interface.switch_port)
vlan_member = vlan_member_out[0].split()[2][4:].strip(",")
except IndexError:
if not _cloud_obj.vlan and not last_nic:
logger.warning(
"Could not determine the previous VLAN member for %s, switch %s, switch port %s "
% (
interface.name,
interface.switch_ip,
interface.switch_port,
))
vlan_member = 0
ssh_helper.disconnect()
if int(old_vlan) != int(vlan):
logger.warning("Interface %s not using QinQ_vl%s",
interface.switch_port, vlan)
if int(vlan_member) != int(vlan):
if not last_nic:
logger.warning(
"Interface %s appears to be a member of VLAN %s, should be %s",
interface.switch_port,
vlan_member,
vlan,
)
if change:
if _cloud_obj.vlan and last_nic:
if int(_cloud_obj.vlan.vlan_id) != int(old_vlan):
logger.info(
f"Change requested for {interface.name}")
logger.info(
"Setting last interface to public vlan %s."
% _cloud_obj.vlan.vlan_id)
juniper = Juniper(
interface.switch_ip,
interface.switch_port,
old_vlan,
vlan,
)
success = juniper.convert_port_public()
else:
logger.info(
f"No changes required for {interface.name}"
)
continue
else:
logger.info(
f"Change requested for {interface.name}")
juniper = Juniper(
interface.switch_ip,
interface.switch_port,
vlan_member,
vlan,
)
success = juniper.set_port()
if success:
logger.info(
"Successfully updated switch settings.")
else:
logger.error(
f"There was something wrong updating switch for {interface.name}"
)
def POST(self, **data):
# make sure post data passed in is ready to pass to mongo engine
result, data = Schedule.prep_data(data)
_start = None
_end = None
if "start" in data:
_start = datetime.datetime.strptime(data["start"],
"%Y-%m-%d %H:%M")
if "end" in data:
_end = datetime.datetime.strptime(data["end"], "%Y-%m-%d %H:%M")
_host = data["host"]
_host_obj = Host.objects(name=_host).first()
broken_hosts = Host.objects(broken=True)
if _host_obj in broken_hosts:
result.append(f"Host {_host_obj.name} is in broken state")
# Check if there were data validation errors
if result:
result = ["Data validation failed: %s" % ", ".join(result)]
cherrypy.response.status = "400 Bad Request"
return json.dumps({"result": result})
cloud_obj = None
if "cloud" in data:
cloud_obj = Cloud.objects(name=data["cloud"]).first()
if not cloud_obj:
result.append("Provided cloud does not exist")
cherrypy.response.status = "400 Bad Request"
return json.dumps({"result": result})
if "index" in data:
data["host"] = _host_obj
schedule = self.model.objects(index=data["index"],
host=data["host"]).first()
if schedule:
if not _start:
_start = schedule.start
if not _end:
_end = schedule.end
if not cloud_obj:
cloud_obj = schedule.cloud
if Schedule.is_host_available(host=_host,
start=_start,
end=_end,
exclude=schedule.index):
data["cloud"] = cloud_obj
notification_obj = Notification.objects(
cloud=cloud_obj, ticket=cloud_obj.ticket).first()
if notification_obj:
notification_obj.update(
one_day=False,
three_days=False,
five_days=False,
seven_days=False,
)
schedule.update(**data)
result.append("Updated %s %s" %
(self.name, schedule.index))
else:
result.append(
"Host is not available during that time frame")
else:
try:
if Schedule.is_host_available(host=_host,
start=_start,
end=_end):
if (self.model.current_schedule(cloud=cloud_obj)
and cloud_obj.validated):
if not cloud_obj.wipe:
_host_obj.update(validated=True)
notification_obj = Notification.objects(
cloud=cloud_obj, ticket=cloud_obj.ticket).first()
if notification_obj:
notification_obj.update(success=False)
schedule = Schedule()
data["cloud"] = cloud_obj
schedule.insert_schedule(**data)
cherrypy.response.status = "201 Resource Created"
result.append("Added schedule for %s on %s" %
(data["host"], cloud_obj.name))
else:
result.append(
"Host is not available during that time frame")
except Exception as e:
# TODO: make sure when this is thrown the output
# points back to here and gives the end user
# enough information to fix the issue
cherrypy.response.status = "500 Internal Server Error"
result.append("Error: %s" % e)
return json.dumps({"result": result})
def main():
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
foreman = Foreman(
conf["foreman_api_url"],
conf["foreman_username"],
conf["foreman_password"],
loop=loop,
)
lines = []
all_hosts = loop.run_until_complete(foreman.get_all_hosts())
blacklist = re.compile("|".join(
[re.escape(word) for word in conf["exclude_hosts"].split("|")]))
broken_hosts = Host.objects(broken=True)
domain_broken_hosts = [
host for host in broken_hosts if conf["domain"] in host.name
]
mgmt_hosts = {}
for host, properties in all_hosts.items():
if not blacklist.search(host):
if properties.get("sp_name", False):
properties["host_ip"] = all_hosts.get(host, {"ip": None})["ip"]
properties["host_mac"] = all_hosts.get(host,
{"mac": None})["mac"]
properties["ip"] = properties.get("sp_ip")
properties["mac"] = properties.get("sp_mac")
mgmt_hosts[properties.get("sp_name")] = properties
lines.append("### **SUMMARY**\n")
_summary = print_summary()
lines.extend(_summary)
details_header = ["\n", "### **DETAILS**\n", "\n"]
lines.extend(details_header)
summary_response = requests.get(os.path.join(API_URL, "summary"))
_cloud_summary = []
if summary_response.status_code == 200:
_cloud_summary = summary_response.json()
for cloud in [cloud for cloud in _cloud_summary if cloud["count"] > 0]:
name = cloud["name"]
owner = cloud["owner"]
lines.append("### <a name=%s></a>\n" % name.strip())
lines.append(
"### **%s : %s (%s) -- %s**\n\n" %
(name.strip(), cloud["count"], cloud["description"], owner))
lines.extend(print_header())
_cloud_obj = Cloud.objects(name=name).first()
_hosts = sorted(
Host.objects(cloud=_cloud_obj, retired=False, broken=False),
key=lambda x: x.name,
)
for host in _hosts:
lines.extend(add_row(host))
lines.append("\n")
lines.extend(print_unmanaged(mgmt_hosts))
lines.extend(print_faulty(domain_broken_hosts))
_full_path = os.path.join(conf["wp_wiki_git_repo_path"], "assignments.md")
if not os.path.exists(conf["wp_wiki_git_repo_path"]):
pathlib.Path(conf["wp_wiki_git_repo_path"]).mkdir(parents=True,
exist_ok=True)
with open(_full_path, "w+") as _f:
_f.seek(0)
for cloud in lines:
_line = cloud if cloud else ""
_f.write(_line)
_f.truncate()
def print_summary():
_summary = []
_headers = [
"**NAME**",
"**SUMMARY**",
"**OWNER**",
"**REQUEST**",
'<span id="status">**STATUS**</span>',
]
if conf["openstack_management"]:
_headers.append("**OSPENV**")
if conf["openshift_management"]:
_headers.append("**OCPINV**")
if conf["gather_ansible_facts"]:
_headers.append("**HWFACTS**")
_summary.append("| %s |\n" % " | ".join(_headers))
_summary.append("| %s |\n" %
" | ".join(["---" for _ in range(len(_headers))]))
_cloud_response = requests.get(os.path.join(API_URL, "summary"))
_cloud_summary = []
if _cloud_response.status_code == 200:
_cloud_summary = _cloud_response.json()
for cloud in [cloud for cloud in _cloud_summary if cloud["count"] > 0]:
cloud_name = cloud["name"]
desc = "%s (%s)" % (cloud["count"], cloud["description"])
owner = cloud["owner"]
ticket = cloud["ticket"]
link = "<a href=%s/%s-%s target=_blank>%s</a>" % (
conf["ticket_url"],
conf["ticket_queue"],
ticket,
ticket,
)
cloud_specific_tag = "%s_%s_%s" % (cloud_name, owner, ticket)
style_tag_end = "</span>"
if cloud["validated"] or cloud_name == "cloud01":
style_tag_start = '<span style="color:green">'
instack_link = os.path.join(conf["quads_url"], "cloud",
"%s_instackenv.json" % cloud_name)
instack_text = "download"
ocpinv_link = os.path.join(conf["quads_url"], "cloud",
"%s_ocpinventory.json" % cloud_name)
ocpinv_text = "download"
status = (
'<span class="progress" style="margin-bottom:0px"><span role="progressbar" aria-valuenow="100" '
'aria-valuemin="0" aria-valuemax="100" style="width:100%" class="progress-bar">100%</span></span> '
)
else:
cloud_obj = Cloud.objects(name=cloud_name).first()
scheduled_hosts = Schedule.current_schedule(
cloud=cloud_obj).count()
moved_hosts = Host.objects(cloud=cloud_obj).count()
percent = moved_hosts / scheduled_hosts * 100
style_tag_start = '<span style="color:red">'
instack_link = "#"
instack_text = "validating"
ocpinv_link = "#"
ocpinv_text = "validating"
if percent < 15:
classes = [
"progress-bar",
"progress-bar-striped",
"progress-bar-danger",
"active",
]
status = (
'<span class="progress" style="margin-bottom:0px"><span role="progressbar" '
'aria-valuenow="100" aria-valuemin="0" aria-valuemax="100" style="width:100%%" '
'class="%s">%.0f%%</span></span>' %
(" ".join(classes), percent))
else:
classes = [
"progress-bar",
"progress-bar-striped",
"progress-bar-warning",
"active",
]
status = (
'<span class="progress" style="margin-bottom:0px"><span role="progressbar" '
'aria-valuenow="%.0f" aria-valuemin="0" aria-valuemax="100" style="width:%.0f%%" '
'class="%s">%.0f%%</span></span>' %
(percent, percent, " ".join(classes), percent))
_data = [
"[%s%s%s](#%s)" %
(style_tag_start, cloud_name, style_tag_end, cloud_name),
desc,
owner,
link,
]
if conf["gather_ansible_facts"]:
factstyle_tag_end = "</span>"
if os.path.exists(
os.path.join(
conf["ansible_facts_web_path"],
"ansible_facts",
"%s_overview.html" % cloud_specific_tag,
)):
factstyle_tag_start = '<span style="color:green">'
ansible_facts_link = os.path.join(
conf["quads_url"],
"ansible_facts",
"%s_overview.html" % cloud_specific_tag,
)
else:
factstyle_tag_start = '<span style="color:red">'
ansible_facts_link = os.path.join(conf["quads_url"],
"underconstruction")
if cloud_name == "cloud01":
_data.append("")
_data.append("")
_data.append(status)
_data.append("")
else:
_data.append("<a href=%s target=_blank>%s%s%s</a>" %
(instack_link, style_tag_start, instack_text,
style_tag_end))
_data.append(
"<a href=%s target=_blank>%s%s%s</a>" %
(ocpinv_link, style_tag_start, ocpinv_text, style_tag_end))
_data.append(status)
_data.append("<a href=%s target=_blank>%sinventory%s</a>" %
(ansible_facts_link, factstyle_tag_start,
factstyle_tag_end))
else:
_data.append(status)
if cloud_name == "cloud01":
if conf["openstack_management"]:
_data.append("")
if conf["openshift_management"]:
_data.append("")
else:
if conf["openstack_management"]:
_data.append("<a href=%s target=_blank>%s%s%s</a>" %
(instack_link, style_tag_start, instack_text,
style_tag_end))
if conf["openshift_management"]:
_data.append("<a href=%s target=_blank>%s%s%s</a>" %
(ocpinv_link, style_tag_start, ocpinv_text,
style_tag_end))
_summary.append("| %s |\n" % " | ".join(_data))
_hosts = Host.objects(broken=False, retired=False)
_host_count = len(_hosts)
_schedules = Schedule.current_schedule().count()
_daily_percentage = _schedules * 100 // _host_count
_summary.append(f"| Total | {_host_count} |\n")
_summary.append("\n")
_summary.append(f"Daily Utilization: {_daily_percentage}% \n")
_summary.append("\n")
_summary.append("[Unmanaged Hosts](#unmanaged)\n")
_summary.append("\n")
_summary.append("[Faulty Hosts](#faulty)\n")
return _summary
def main():
loop = asyncio.get_event_loop()
foreman_admin = Foreman(
conf["foreman_api_url"],
conf["foreman_username"],
conf["foreman_password"],
loop=loop,
)
ignore = ["cloud01"]
foreman_rbac_exclude = conf.get("foreman_rbac_exclude")
if foreman_rbac_exclude:
ignore.extend(foreman_rbac_exclude.split("|"))
clouds = Cloud.objects()
for cloud in clouds:
infra_pass = f"{conf['infra_location']}@{cloud.ticket}"
loop.run_until_complete(
foreman_admin.update_user_password(cloud.name, infra_pass)
)
foreman_cloud_user = Foreman(
conf["foreman_api_url"],
cloud.name,
infra_pass,
loop=loop,
)
if cloud.name not in ignore:
logger.info(f"Processing {cloud.name}")
cloud_hosts = loop.run_until_complete(foreman_cloud_user.get_all_hosts())
user_id = loop.run_until_complete(foreman_admin.get_user_id(cloud.name))
admin_id = loop.run_until_complete(
foreman_admin.get_user_id(conf["foreman_username"])
)
current_schedule = Schedule.current_schedule(cloud=cloud)
if current_schedule:
logger.info(f" Current Host Permissions:")
for host, properties in cloud_hosts.items():
logger.info(f" {host}")
match = [
schedule.host.name
for schedule in current_schedule
if schedule.host.name == host
]
if not match:
_host_id = loop.run_until_complete(
foreman_admin.get_host_id(host)
)
loop.run_until_complete(
foreman_admin.put_element(
"hosts", _host_id, "owner_id", admin_id
)
)
logger.info(f"* Removed permission {host}")
for schedule in current_schedule:
match = [
host
for host, _ in cloud_hosts.items()
if host == schedule.host.name
]
if not match:
# want to run these separately to avoid ServerDisconnect
_host_id = loop.run_until_complete(
foreman_admin.get_host_id(schedule.host.name)
)
loop.run_until_complete(
foreman_admin.put_element(
"hosts", _host_id, "owner_id", user_id
)
)
logger.info(f"* Added permission {schedule.host.name}")
else:
if cloud_hosts:
logger.info(" No active schedule, removing pre-existing roles.")
for host, properties in cloud_hosts.items():
_host_id = loop.run_until_complete(
foreman_admin.get_host_id(host)
)
loop.run_until_complete(
foreman_admin.put_element(
"hosts", _host_id, "owner_id", admin_id
)
)
logger.info(f"* Removed permission {host}")
else:
logger.info(" No active schedule nor roles assigned.")