def post_system_test(_cloud): foreman = Foreman(conf["foreman_api_url"], _cloud.name, _cloud.ticket) quads = Api(API_URL) try: build_hosts = foreman.get_build_hosts() except RequestException: logger.error("Unable to query Foreman for cloud: %s" % _cloud.name) logger.error("Verify Foreman password is correct: %s" % _cloud.ticket) return False pending = [] schedules = quads.get_current_schedule(cloud=_cloud.name) if "result" not in schedules: for schedule in schedules: host = quads.get_hosts(id=schedule["host"]["$oid"]) if host and host['name'] in build_hosts: pending.append(host["name"]) if pending: logger.info("The following hosts are marked for build:") for host in pending: logger.info(host) return False return True
def post_system_test(self): foreman = Foreman( conf["foreman_api_url"], self.cloud.name, self.cloud.ticket ) quads = Api(API_URL) if not foreman.verify_credentials(): logger.error("Unable to query Foreman for cloud: %s" % self.cloud.name) logger.error("Verify Foreman password is correct: %s" % self.cloud.ticket) self.report = self.report + "Unable to query Foreman for cloud: %s\n" % self.cloud.name self.report = self.report + "Verify Foreman password is correct: %s\n" % self.cloud.ticket return False build_hosts = foreman.get_build_hosts() pending = [] schedules = quads.get_current_schedule(cloud=self.cloud.name) if "result" not in schedules: for schedule in schedules: host = quads.get_hosts(id=schedule["host"]["$oid"]) if host and host['name'] in build_hosts: pending.append(host["name"]) if pending: logger.info("The following hosts are marked for build:") self.report = self.report + "The following hosts are marked for build:\n" for host in pending: logger.info(host) self.report = self.report + "%s\n" % host return False return True
class TestForeman(object): def __init__(self): self.foreman = None def setup(self): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self.foreman = Foreman( conf["foreman_api_url"], conf["ipmi_username"], conf["ipmi_password"], loop=loop, ) def teardown(self): self.foreman.loop.close() def test_get_all_hosts(self): hosts = self.foreman.loop.run_until_complete( self.foreman.get_all_hosts()) assert isinstance(hosts, dict) def test_get_broken_hosts(self): hosts = self.foreman.loop.run_until_complete( self.foreman.get_broken_hosts()) assert isinstance(hosts, dict)
def setup(self): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self.foreman = Foreman( conf["foreman_api_url"], conf["ipmi_username"], conf["ipmi_password"], loop=loop, )
def main(): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) foreman = Foreman( Config["foreman_api_url"], Config["foreman_username"], Config["foreman_password"], loop=loop, ) all_hosts = loop.run_until_complete(foreman.get_all_hosts()) blacklist = re.compile("|".join( [re.escape(word) for word in Config["exclude_hosts"].split("|")])) hosts = {} for host, properties in all_hosts.items(): if not blacklist.search(host): if properties.get("sp_name", False): properties["host_ip"] = properties["ip"] properties["host_mac"] = properties["mac"] properties["ip"] = properties.get("sp_ip") properties["mac"] = properties.get("sp_mac") consolidate_ipmi_data(host, "macaddr", properties["host_mac"]) consolidate_ipmi_data(host, "oobmacaddr", properties.get("sp_mac")) svctag_file = os.path.join(Config["data_dir"], "ipmi", host, "svctag") svctag = "" if os.path.exists(svctag_file): with open(svctag_file) as _file: svctag = _file.read() properties["svctag"] = svctag.strip() hosts[host] = properties _full_path = os.path.join(Config["wp_wiki_git_repo_path"], "main.md") if not os.path.exists(Config["wp_wiki_git_repo_path"]): pathlib.Path(Config["wp_wiki_git_repo_path"]).mkdir(parents=True, exist_ok=True) with open(_full_path, "w") as _f: _f.seek(0) for rack in Config["racks"].split(): if rack_has_hosts(rack, hosts): _f.write(render_header(rack)) for host, properties in hosts.items(): if rack in host: host_obj = Host.objects(name=host).first() if host_obj and not host_obj.retired: _f.write(render_row(host_obj, properties)) _f.write("\n") _f.truncate()
def main(): foreman = Foreman( conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"], ) all_hosts = foreman.get_all_hosts() blacklist = re.compile("|".join( [re.escape(word) for word in conf["exclude_hosts"].split("|")])) hosts = {} for host, properties in all_hosts.items(): if not blacklist.search(host): mgmt_host = foreman.get_idrac_host_with_details(host) if mgmt_host: properties["host_ip"] = properties["ip"] properties["host_mac"] = properties["mac"] properties["ip"] = mgmt_host["ip"] properties["mac"] = mgmt_host["mac"] consolidate_ipmi_data(host, "macaddr", properties["host_mac"]) consolidate_ipmi_data(host, "oobmacaddr", mgmt_host["mac"]) svctag_file = os.path.join(conf["data_dir"], "ipmi", host, "svctag") svctag = "" if os.path.exists(svctag_file): with open(svctag_file) as _file: svctag = _file.read() properties["svctag"] = svctag.strip() hosts[host] = properties _full_path = os.path.join(conf["wp_wiki_git_repo_path"], "main.md") if not os.path.exists(conf["wp_wiki_git_repo_path"]): pathlib.Path(conf["wp_wiki_git_repo_path"]).mkdir(parents=True, exist_ok=True) with open(_full_path, "w") as _f: _f.seek(0) for rack in conf["racks"].split(): if rack_has_hosts(rack, hosts): _f.write(render_header(rack)) for host, properties in hosts.items(): if rack in host: _f.write(render_row(host, properties)) _f.write("\n") _f.truncate()
class TestForeman(object): def setup(self): self.foreman = Foreman( conf["foreman_api_url"], conf["ipmi_username"], conf["ipmi_password"], ) def test_get_all_hosts(self): hosts = self.foreman.get_all_hosts() assert isinstance(hosts, dict) def test_get_broken_hosts(self): hosts = self.foreman.get_broken_hosts() assert isinstance(hosts, dict)
def available(search): models = search.data['model'] if models: query = None for model in models: if query: query = query | Q(model=model.upper()) else: query = Q(model=model.upper()) hosts = Host.objects.filter(query) else: hosts = Host.objects().all() loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) foreman = Foreman( conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"], loop=loop, ) broken_hosts = loop.run_until_complete(foreman.get_broken_hosts()) available_hosts = [] start = datetime.combine(search.data['start'], time.min) end = datetime.combine(search.data['end'], time.min) if hosts: for host in hosts: if Schedule.is_host_available( host=host["name"], start=start, end=end) and not broken_hosts.get(host["name"], False): host_dict = {"name": host.name, "model": host.model} available_hosts.append(host_dict) return jsonify(available_hosts)
def setup(self): self.foreman = Foreman( conf["foreman_api_url"], conf["ipmi_username"], conf["ipmi_password"], )
async def move_and_rebuild(host, new_cloud, semaphore, rebuild=False, loop=None): build_start = datetime.now() logger.debug("Moving and rebuilding host: %s" % host) untouchable_hosts = Config["untouchable_hosts"] logger.debug("Untouchable hosts: %s" % untouchable_hosts) _host_obj = Host.objects(name=host).first() if host in untouchable_hosts: logger.error("No way...") return False _target_cloud = Cloud.objects(name=new_cloud).first() ipmi_new_pass = (f"{Config['infra_location']}@{_target_cloud.ticket}" if _target_cloud.ticket else Config["ipmi_password"]) ipmi_set_pass = [ "user", "set", "password", str(Config["ipmi_cloud_username_id"]), ipmi_new_pass, ] new_semaphore = asyncio.Semaphore(20) await execute_ipmi(host, arguments=ipmi_set_pass, semaphore=new_semaphore) ipmi_set_operator = [ "user", "priv", str(Config["ipmi_cloud_username_id"]), "0x4" ] await execute_ipmi(host, arguments=ipmi_set_operator, semaphore=new_semaphore) badfish = None if rebuild and _target_cloud.name != _host_obj.default_cloud.name: if Config.pdu_management: # TODO: pdu management pass try: badfish = await badfish_factory( "mgmt-%s" % host, Config["ipmi_username"], Config["ipmi_password"], propagate=True, ) except BadfishException: logger.error( f"Could not initialize Badfish. Verify ipmi credentials for mgmt-{host}." ) return False if is_supported(host): try: interfaces_path = os.path.join( os.path.dirname(__file__), "../../conf/idrac_interfaces.yml") await badfish.change_boot("director", interfaces_path) # wait 10 minutes for the boot order job to complete await asyncio.sleep(600) except BadfishException: logger.error( f"Could not set boot order via Badfish for mgmt-{host}.") return False try: await badfish.set_power_state("on") except BadfishException: logger.error(f"Failed to power on {host}") return False foreman_results = [] params = [ { "name": "operatingsystems", "value": Config["foreman_default_os"], "identifier": "title", }, { "name": "ptables", "value": Config["foreman_default_ptable"] }, { "name": "media", "value": Config["foreman_default_medium"] }, ] foreman = Foreman( Config["foreman_api_url"], Config["foreman_username"], Config["foreman_password"], semaphore=semaphore, loop=loop, ) set_result = await foreman.set_host_parameter(host, "overcloud", "true") foreman_results.append(set_result) put_result = await foreman.put_parameter(host, "build", 1) foreman_results.append(put_result) put_param_result = await foreman.put_parameters_by_name(host, params) foreman_results.append(put_param_result) owner_id = await foreman.get_user_id(new_cloud) host_id = await foreman.get_host_id(host) put_result = await foreman.put_element("hosts", host_id, "owner_id", owner_id) foreman_results.append(put_result) for result in foreman_results: if isinstance(result, Exception) or not result: logger.error( "There was something wrong setting Foreman host parameters." ) return False if is_supported(host): try: await badfish.boot_to_type( "foreman", os.path.join(os.path.dirname(__file__), "../../conf/idrac_interfaces.yml"), ) await badfish.reboot_server(graceful=False) except BadfishException: logger.error(f"Error setting PXE boot via Badfish on {host}.") await badfish.reboot_server(graceful=False) return False else: try: asyncio.run_coroutine_threadsafe( badfish.unmount_virtual_media(), loop, ) except BadfishException: logger.warning( f"Could not unmount virtual media for mgmt-{host}.") try: ipmi_pxe_persistent = [ "chassis", "bootdev", "pxe", "options=persistent", ] await execute_ipmi(host, arguments=ipmi_pxe_persistent, semaphore=new_semaphore) await ipmi_reset(host, new_semaphore) except Exception as ex: logger.debug(ex) logger.error( f"There was something wrong setting PXE flag or resetting IPMI on {host}." ) if _target_cloud.name == _host_obj.default_cloud.name: if not badfish: try: badfish = await badfish_factory( "mgmt-%s" % host, Config["ipmi_username"], Config["ipmi_password"], propagate=True, ) except BadfishException: logger.error( f"Could not initialize Badfish. Verify ipmi credentials for mgmt-{host}." ) return False await badfish.set_power_state("off") source_cloud_schedule = Schedule.current_schedule( cloud=_host_obj.cloud.name) if not source_cloud_schedule: _old_cloud_obj = Cloud.objects(name=_host_obj.cloud.name).first() _old_cloud_obj.update(vlan=None) schedule = Schedule.current_schedule(cloud=_target_cloud, host=_host_obj).first() if schedule: schedule.update(build_start=build_start, build_end=datetime.now()) schedule.save() logger.debug("Updating host: %s") _host_obj.update(cloud=_target_cloud, build=False, last_build=datetime.now(), validated=False) return True
def GET(self, **data): args = {} _cloud = None _host = None if "cloudonly" in data: _cloud = model.Cloud.objects(cloud=data["cloudonly"]) if not _cloud: cherrypy.response.status = "404 Not Found" return json.dumps({"result": "Cloud %s Not Found" % data["cloudonly"]}) else: return _cloud.to_json() if self.name == "host": if "id" in data: _host = model.Host.objects(id=data["id"]).first() elif "name" in data: _host = model.Host.objects(name=data["name"]).first() elif "cloud" in data: _cloud = model.Cloud.objects(name=data["cloud"]).first() _host = model.Host.objects(cloud=_cloud) else: _host = model.Host.objects() if not _host: return json.dumps({"result": ["Nothing to do."]}) return _host.to_json() if self.name == "ccuser": _clouds = self.model.objects().all() clouds_summary = [] for cloud in _clouds: count = model.Schedule.current_schedule(cloud=cloud).count() clouds_summary.append( { "name": cloud.name, "count": count, "description": cloud.description, "owner": cloud.owner, "ticket": cloud.ticket, "ccuser": cloud.ccuser, "provisioned": cloud.provisioned, } ) return json.dumps(clouds_summary) if self.name == "cloud": if "id" in data: _cloud = model.Cloud.objects(id=data["id"]).first() elif "name" in data: _cloud = model.Cloud.objects(name=data["name"]).first() elif "owner" in data: _cloud = model.Cloud.to_json(owner=data["owner"]).first() if _cloud: return _cloud.to_json() if self.name == "available": _start = _end = datetime.datetime.now() if "start" in data: _start = datetime.datetime.strptime(data["start"], "%Y-%m-%dT%H:%M:%S") if "end" in data: _end = datetime.datetime.strptime(data["end"], "%Y-%m-%dT%H:%M:%S") available = [] all_hosts = model.Host.objects().all() loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) foreman = Foreman( conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"], loop=loop, ) broken_hosts = loop.run_until_complete(foreman.get_broken_hosts()) for host in all_hosts: if model.Schedule.is_host_available( host=host["name"], start=_start, end=_end ) and not broken_hosts.get(host["name"], False): available.append(host["name"]) return json.dumps(available) if self.name == "summary": _clouds = model.Cloud.objects().all() clouds_summary = [] total_count = 0 for cloud in _clouds: if cloud.name == "cloud01": count = model.Host.objects(cloud=cloud).count() else: date = datetime.datetime.now() if "date" in data: date = datetime.datetime.strptime(data["date"], "%Y-%m-%dT%H:%M:%S") count = self.model.current_schedule(cloud=cloud, date=date).count() total_count += count clouds_summary.append( { "name": cloud.name, "count": count, "description": cloud.description, "owner": cloud.owner, "ticket": cloud.ticket, "ccuser": cloud.ccuser, "provisioned": cloud.provisioned, "validated": cloud.validated, } ) if "date" in data: host_count = model.Host.objects().count() for cloud in clouds_summary: if cloud["name"] == "cloud01": cloud["count"] = host_count - total_count return json.dumps(clouds_summary) if self.name == "qinq": _clouds = model.Cloud.objects().all() clouds_qinq = [] for cloud in _clouds: if cloud.qinq: qinq_value = "0 (Isolated)" else: qinq_value = "1 (Combined)" clouds_qinq.append( { "name": cloud.name, "qinq": qinq_value, } ) return json.dumps(clouds_qinq) objs = self.model.objects(**args) if objs: return objs.to_json() else: return json.dumps({"result": ["No results."]})
def make_env_json(filename): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) foreman = Foreman( conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"], loop=loop, ) cloud_list = Cloud.objects() if not os.path.exists(conf["json_web_path"]): os.makedirs(conf["json_web_path"]) now = time.time() old_jsons = [ file for file in os.listdir(conf["json_web_path"]) if ":" in file ] for file in old_jsons: if (os.stat(os.path.join(conf["json_web_path"], file)).st_mtime < now - conf["json_retention_days"] * 86400): os.remove(os.path.join(conf["json_web_path"], file)) for cloud in cloud_list: host_list = Host.objects(cloud=cloud).order_by("name") foreman_password = conf["ipmi_password"] if cloud.ticket: foreman_password = f"{conf['infra_location']}@{cloud.ticket}" data = defaultdict(list) for host in host_list: if conf["foreman_unavailable"]: overcloud = {"result": "true"} else: overcloud = loop.run_until_complete( foreman.get_host_param(host.name, "overcloud")) if not overcloud: overcloud = {"result": "true"} if type(overcloud["result"]) != bool: try: _overcloud_result = strtobool(overcloud["result"]) except ValueError: print( f"WARN: {host.name} overcloud value is not set correctly." ) _overcloud_result = 1 else: _overcloud_result = overcloud["result"] if "result" in overcloud and _overcloud_result: mac = [] if filename == "instackenv": for interface in host.interfaces: if interface.pxe_boot: mac.append(interface.mac_address) if filename == "ocpinventory": mac = [ interface.mac_address for interface in host.interfaces ] data["nodes"].append({ "pm_password": foreman_password, "pm_type": "pxe_ipmitool", "mac": mac, "cpu": "2", "memory": "1024", "disk": "20", "arch": "x86_64", "pm_user": conf["ipmi_cloud_username"], "pm_addr": "mgmt-%s" % host.name, }) content = json.dumps(data, indent=4, sort_keys=True) if not os.path.exists(conf["json_web_path"]): pathlib.Path(conf["json_web_path"]).mkdir(parents=True, exist_ok=True) now = datetime.now() new_json_file = os.path.join( conf["json_web_path"], "%s_%s.json_%s" % (cloud.name, filename, now.strftime("%Y-%m-%d_%H:%M:%S")), ) json_file = os.path.join(conf["json_web_path"], "%s_%s.json" % (cloud.name, filename)) with open(new_json_file, "w+") as _json_file: _json_file.seek(0) _json_file.write(content) os.chmod(new_json_file, 0o644) copyfile(new_json_file, json_file)
def main(): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) foreman = Foreman( conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"], loop=loop, ) lines = [] all_hosts = loop.run_until_complete(foreman.get_all_hosts()) blacklist = re.compile("|".join( [re.escape(word) for word in conf["exclude_hosts"].split("|")])) broken_hosts = Host.objects(broken=True) domain_broken_hosts = [ host for host in broken_hosts if conf["domain"] in host.name ] mgmt_hosts = {} for host, properties in all_hosts.items(): if not blacklist.search(host): if properties.get("sp_name", False): properties["host_ip"] = all_hosts.get(host, {"ip": None})["ip"] properties["host_mac"] = all_hosts.get(host, {"mac": None})["mac"] properties["ip"] = properties.get("sp_ip") properties["mac"] = properties.get("sp_mac") mgmt_hosts[properties.get("sp_name")] = properties lines.append("### **SUMMARY**\n") _summary = print_summary() lines.extend(_summary) details_header = ["\n", "### **DETAILS**\n", "\n"] lines.extend(details_header) summary_response = requests.get(os.path.join(API_URL, "summary")) _cloud_summary = [] if summary_response.status_code == 200: _cloud_summary = summary_response.json() for cloud in [cloud for cloud in _cloud_summary if cloud["count"] > 0]: name = cloud["name"] owner = cloud["owner"] lines.append("### <a name=%s></a>\n" % name.strip()) lines.append( "### **%s : %s (%s) -- %s**\n\n" % (name.strip(), cloud["count"], cloud["description"], owner)) lines.extend(print_header()) _cloud_obj = Cloud.objects(name=name).first() _hosts = sorted( Host.objects(cloud=_cloud_obj, retired=False, broken=False), key=lambda x: x.name, ) for host in _hosts: lines.extend(add_row(host)) lines.append("\n") lines.extend(print_unmanaged(mgmt_hosts)) lines.extend(print_faulty(domain_broken_hosts)) _full_path = os.path.join(conf["wp_wiki_git_repo_path"], "assignments.md") if not os.path.exists(conf["wp_wiki_git_repo_path"]): pathlib.Path(conf["wp_wiki_git_repo_path"]).mkdir(parents=True, exist_ok=True) with open(_full_path, "w+") as _f: _f.seek(0) for cloud in lines: _line = cloud if cloud else "" _f.write(_line) _f.truncate()
def main(): foreman = Foreman(conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"]) _cloud_response = requests.get(os.path.join(API_URL, "cloud")) cloud_list = [] if _cloud_response.status_code == 200: cloud_list = _cloud_response.json() _host_response = requests.get(os.path.join(API_URL, "host")) host_list = [] if _host_response.status_code == 200: host_list = _host_response.json() if not os.path.exists(conf["json_web_path"]): os.makedirs(conf["json_web_path"]) old_jsons = [ file for file in os.listdir(conf["json_web_path"]) if ".json" in file ] for file in old_jsons: os.remove(os.path.join(conf["json_web_path"], file)) over_cloud = foreman.get_parametrized( "params.%s" % conf["foreman_director_parameter"], "true") columns = [ "macaddress", "ipmi url", "ipmi user", "ipmi password", "ipmi tool" ] lines = [] for cloud in cloud_list: lines.append(",".join(columns)) tickets = cloud["ticket"] if tickets: foreman_password = tickets[0] else: foreman_password = conf["ipmi_password"] for host in host_list: is_overcloud = host["name"] in over_cloud.keys() if is_overcloud: mac = over_cloud[host]["mac"] ipmi_url = "mgmt-%s" % host["name"] ipmi_username = conf["ipmi_cloud_username"] ipmi_tool = "pxe_ipmitool" line = ",".join([ mac, ipmi_url, ipmi_username, foreman_password, ipmi_tool ]) lines.append(line) with NamedTemporaryFile() as ntp: for line in lines: new_line = "%s\n" % line ntp.write(bytes(new_line.encode("utf-8"))) ntp.seek(0) content = csv_to_instack(ntp.name) if not os.path.exists(conf["json_web_path"]): pathlib.Path(conf["json_web_path"]).mkdir(parents=True, exist_ok=True) json_file = os.path.join(conf["json_web_path"], "%s_instackenv.json" % cloud['name']) now = datetime.now() if os.path.exists(json_file): os.rename( json_file, "%s_%s" % (json_file, now.strftime("%Y-%m-%d_%H:%M:%S"))) with open(json_file, "w+") as _json_file: _json_file.seek(0) _json_file.write(content) os.chmod(json_file, 644)
def main(): if conf["openstack_management"]: foreman = Foreman(conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"]) cloud_list = Cloud.objects() if not os.path.exists(conf["json_web_path"]): os.makedirs(conf["json_web_path"]) now = time.time() old_jsons = [ file for file in os.listdir(conf["json_web_path"]) if ":" in file ] for file in old_jsons: if os.stat(os.path.join(conf["json_web_path"], file) ).st_mtime < now - conf["json_retention_days"] * 86400: os.remove(os.path.join(conf["json_web_path"], file)) for cloud in cloud_list: host_list = Host.objects(cloud=cloud).order_by("name") foreman_password = conf["ipmi_password"] if cloud.ticket: foreman_password = cloud.ticket json_data = defaultdict(list) for host in host_list[1:]: if conf["foreman_unavailable"]: overcloud = {"result": "true"} else: overcloud = foreman.get_host_param(host.name, "overcloud") if not overcloud: overcloud = {"result": "true"} if "result" in overcloud and strtobool(overcloud["result"]): mac = "00:00:00:00:00:00" if len(host.interfaces) > 1: mac = host.interfaces[1].mac_address json_data['nodes'].append({ 'pm_password': foreman_password, 'pm_type': "pxe_ipmitool", 'mac': [mac], 'cpu': "2", 'memory': "1024", 'disk': "20", 'arch': "x86_64", 'pm_user': conf["ipmi_cloud_username"], 'pm_addr': "mgmt-%s" % host.name }) content = json.dumps(json_data, indent=4, sort_keys=True) if not os.path.exists(conf["json_web_path"]): pathlib.Path(conf["json_web_path"]).mkdir(parents=True, exist_ok=True) now = datetime.now() new_json_file = os.path.join( conf["json_web_path"], "%s_instackenv.json_%s" % (cloud.name, now.strftime("%Y-%m-%d_%H:%M:%S"))) json_file = os.path.join(conf["json_web_path"], "%s_instackenv.json" % cloud.name) with open(new_json_file, "w+") as _json_file: _json_file.seek(0) _json_file.write(content) os.chmod(new_json_file, 0o644) copyfile(new_json_file, json_file)
def move_and_rebuild(host, old_cloud, new_cloud, rebuild=False): logger.debug("Moving and rebuilding host: %s" % host) untouchable_hosts = conf["untouchable_hosts"] logger.debug("Untouchable hosts: %s" % untouchable_hosts) _host_obj = Host.objects(name=host).first() if not _host_obj.interfaces: logger.error("Host has no interfaces defined.") return False if host in untouchable_hosts: logger.error("No way...") return False _old_cloud_obj = Cloud.objects(name=old_cloud).first() _new_cloud_obj = Cloud.objects(name=new_cloud).first() logger.debug("Connecting to switch on: %s" % _host_obj.interfaces[0].ip_address) _public_vlan_obj = Vlan.objects(cloud=_new_cloud_obj).first() for i, interface in enumerate(_host_obj.interfaces): ssh_helper = SSHHelper(interface.ip_address, conf["junos_username"]) old_vlan_out = ssh_helper.run_cmd("show configuration interfaces %s" % interface.switch_port) old_vlan = old_vlan_out[0].split(";")[0].split()[1][7:] if not old_vlan: logger.warning( "Warning: Could not determine the previous VLAN for %s on %s, switch %s, switchport %s" % (host, interface.name, interface.ip_address, interface.switch_port)) old_vlan = get_vlan(_old_cloud_obj, i) new_vlan = get_vlan(_new_cloud_obj, i) if _public_vlan_obj and i == len(_host_obj.interfaces) - 1: logger.info("Setting last interface to public vlan %s." % new_vlan) if int(old_vlan) != int(_public_vlan_obj.vlan_id): success = juniper_convert_port_public( interface.ip_address, interface.switch_port, str(old_vlan), str(_public_vlan_obj.vlan_id)) if success: logger.info("Successfully updated switch settings.") else: logger.error( "There was something wrong updating switch for %s:%s" % (host, interface.name)) return False else: if int(old_vlan) != int(new_vlan): success = juniper_set_port(interface.ip_address, interface.switch_port, str(old_vlan), str(new_vlan)) if success: logger.info("Successfully update switch settings.") else: logger.error( "There was something wrong updating switch for %s:%s" % (host, interface.name)) return False if i == len(_host_obj.interfaces) - 1: _old_vlan_obj = Vlan.objects(cloud=_old_cloud_obj).first() if _old_vlan_obj: _old_vlan_obj.update(cloud=None) ssh_helper.disconnect() ipmi_new_pass = _new_cloud_obj.ticket if _new_cloud_obj.ticket else conf[ "$ipmi_password"] foreman = Foreman( conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"], ) foreman.remove_role(_old_cloud_obj.name, _host_obj.name) foreman.add_role(_new_cloud_obj.name, _host_obj.name) foreman.update_user_password(_new_cloud_obj.name, ipmi_new_pass) ipmi_set_pass = [ "/usr/bin/ipmitool", "-I", "lanplus", "-H", "mgmt-%s" % host, "-U", conf["ipmi_username"], "-P", conf["ipmi_password"], "user", "set", "password", str(conf["ipmi_cloud_username_id"]), ipmi_new_pass ] logger.debug("ipmi_set_pass: %s" % ipmi_set_pass) subprocess.call(ipmi_set_pass) ipmi_set_operator = [ "/usr/bin/ipmitool", "-I", "lanplus", "-H", "mgmt-%s" % host, "-U", conf["ipmi_username"], "-P", conf["ipmi_password"], "user", "priv", str(conf["ipmi_cloud_username_id"]), "0x4" ] logger.debug("ipmi_set_operator: %s" % ipmi_set_operator) subprocess.call(ipmi_set_operator) if rebuild and _new_cloud_obj.name != "cloud01": badfish = Badfish("mgmt-%s" % host, conf["ipmi_username"], conf["ipmi_password"]) if "pdu_management" in conf and conf["pdu_management"]: # TODO: pdu management pass if is_supermicro(host): ipmi_pxe_persistent = [ "/usr/bin/ipmitool", "-I", "lanplus", "-H", "mgmt-%s" % host, "-U", conf["ipmi_username"], "-P", conf["ipmi_password"], "chassis", "bootdev", "pxe", "options", "=", "persistent" ] logger.debug("ipmi_pxe_persistent: %s" % ipmi_pxe_persistent) subprocess.call(ipmi_pxe_persistent) if is_supported(host): try: badfish.change_boot( "director", os.path.join(os.path.dirname(__file__), "../../conf/idrac_interfaces.yml")) except SystemExit as ex: logger.debug(ex) logger.error("Could not set boot order via Badfish.") return False foreman_success = foreman.remove_extraneous_interfaces(host) foreman_success = foreman.set_host_parameter( host, "rhel73", "false") and foreman_success foreman_success = foreman.set_host_parameter( host, "rhel75", "false") and foreman_success foreman_success = foreman.set_host_parameter( host, "overcloud", "true") and foreman_success foreman_success = foreman.put_parameter(host, "build", 1) and foreman_success foreman_success = foreman.put_parameter_by_name( host, "operatingsystems", conf["foreman_default_os"], "title") and foreman_success foreman_success = foreman.put_parameter_by_name( host, "ptables", conf["foreman_default_ptable"]) and foreman_success foreman_success = foreman.put_parameter_by_name( host, "media", conf["foreman_default_medium"]) and foreman_success if not foreman_success: logger.error( "There was something wrong setting Foreman host parameters.") try: badfish.set_next_boot_pxe() badfish.reboot_server() except SystemExit: if is_supermicro(host): logger.warning("Badfish not yet supported on Supermicro: %s." % host) else: logger.exception( "There was something wrong setting next PXE boot via Badfish." ) return False logger.debug("Updating host: %s") _host_obj.update(cloud=_new_cloud_obj, build=False, last_build=datetime.now()) else: logger.debug("Updating host: %s") _host_obj.update(cloud=_new_cloud_obj, build=False, last_build=datetime.now())
def main(): foreman = Foreman(conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"]) lines = [] all_hosts = foreman.get_all_hosts() blacklist = re.compile("|".join( [re.escape(word) for word in conf["exclude_hosts"].split("|")])) broken_hosts = foreman.get_broken_hosts() domain_broken_hosts = { host: properties for host, properties in broken_hosts.items() if conf["domain"] in host } mgmt_hosts = {} for host, properties in all_hosts.items(): if not blacklist.search(host): mgmt_host = foreman.get_idrac_host_with_details(host) if mgmt_host: properties["host_ip"] = all_hosts.get(host, {"ip": None})["ip"] properties["host_mac"] = all_hosts.get(host, {"mac": None})["mac"] properties["ip"] = mgmt_host["ip"] properties["mac"] = mgmt_host["mac"] mgmt_hosts[mgmt_host["name"]] = properties lines.append("### **SUMMARY**\n") _summary = print_summary() lines.extend(_summary) details_header = ["\n", "### **DETAILS**\n", "\n"] lines.extend(details_header) summary_response = requests.get(os.path.join(API_URL, "summary")) _cloud_summary = [] if summary_response.status_code == 200: _cloud_summary = summary_response.json() _cloud_hosts = [] for cloud in [cloud for cloud in _cloud_summary if cloud["count"] > 0]: name = cloud["name"] owner = cloud["owner"] lines.append("### <a name=%s></a>\n" % name.strip()) lines.append("### **%s -- %s**\n\n" % (name.strip(), owner)) lines.extend(print_header()) _cloud_hosts = requests.get( os.path.join(API_URL, "host?cloud=%s" % name)) if _cloud_hosts.status_code == 200: for host in _cloud_hosts.json(): if "cloud" in host: lines.extend(add_row(host)) lines.append("\n") lines.extend(print_unmanaged(mgmt_hosts, domain_broken_hosts)) lines.extend(print_faulty(domain_broken_hosts)) _full_path = os.path.join(conf["wp_wiki_git_repo_path"], "assignments.md") if not os.path.exists(conf["wp_wiki_git_repo_path"]): pathlib.Path(conf["wp_wiki_git_repo_path"]).mkdir(parents=True, exist_ok=True) with open(_full_path, "w+") as _f: _f.seek(0) for cloud in lines: _line = cloud if cloud else "" _f.write(_line) _f.truncate()
async def move_and_rebuild(host, new_cloud, semaphore, rebuild=False, loop=None): build_start = datetime.now() logger.debug("Moving and rebuilding host: %s" % host) untouchable_hosts = conf["untouchable_hosts"] logger.debug("Untouchable hosts: %s" % untouchable_hosts) _host_obj = Host.objects(name=host).first() if host in untouchable_hosts: logger.error("No way...") return False _new_cloud_obj = Cloud.objects(name=new_cloud).first() ipmi_new_pass = (f"{conf['infra_location']}@{_new_cloud_obj.ticket}" if _new_cloud_obj.ticket else conf["ipmi_password"]) ipmi_set_pass = [ "user", "set", "password", str(conf["ipmi_cloud_username_id"]), ipmi_new_pass, ] new_semaphore = asyncio.Semaphore(20) await execute_ipmi(host, arguments=ipmi_set_pass, semaphore=new_semaphore) ipmi_set_operator = [ "user", "priv", str(conf["ipmi_cloud_username_id"]), "0x4" ] await execute_ipmi(host, arguments=ipmi_set_operator, semaphore=new_semaphore) if rebuild and _new_cloud_obj.name != _host_obj.default_cloud.name: if "pdu_management" in conf and conf["pdu_management"]: # TODO: pdu management pass if is_supported(host): try: badfish = await badfish_factory( "mgmt-%s" % host, conf["ipmi_username"], conf["ipmi_password"], propagate=True, ) except BadfishException: logger.error( f"Could not initialize Badfish. Verify ipmi credentials for mgmt-{host}." ) return False try: changed_boot_order = asyncio.run_coroutine_threadsafe( badfish.change_boot( "director", os.path.join(os.path.dirname(__file__), "../../conf/idrac_interfaces.yml"), ), loop, ) if changed_boot_order: await badfish.reboot_server(graceful=False) except BadfishException: logger.error( f"Could not set boot order via Badfish for mgmt-{host}.") return False foreman_results = [] params = [ { "name": "operatingsystems", "value": conf["foreman_default_os"], "identifier": "title", }, { "name": "ptables", "value": conf["foreman_default_ptable"] }, { "name": "media", "value": conf["foreman_default_medium"] }, ] foreman = Foreman( conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"], semaphore=semaphore, loop=loop, ) set_result = await foreman.set_host_parameter(host, "overcloud", "true") foreman_results.append(set_result) put_result = await foreman.put_parameter(host, "build", 1) foreman_results.append(put_result) put_param_result = await foreman.put_parameters_by_name(host, params) foreman_results.append(put_param_result) owner_id = await foreman.get_user_id(new_cloud) host_id = await foreman.get_host_id(host) put_result = await foreman.put_element("hosts", host_id, "owner_id", owner_id) foreman_results.append(put_result) for result in foreman_results: if isinstance(result, Exception) or not result: logger.error( "There was something wrong setting Foreman host parameters." ) return False healthy = False for i in range(RETRIES): nc = Netcat(_host_obj.name) if nc.health_check(): healthy = True nc.close() break nc.close() if not healthy: logger.error("Failed to recover host after changing boot order.") return False if is_supported(host): try: await badfish.boot_to_type( "foreman", os.path.join(os.path.dirname(__file__), "../../conf/idrac_interfaces.yml"), ) await badfish.reboot_server(graceful=False) except BadfishException: logger.error(f"Error setting PXE boot via Badfish on {host}.") await badfish.reboot_server(graceful=False) return False else: try: ipmi_pxe_persistent = [ "chassis", "bootdev", "pxe", "options", "=", "persistent", ] await execute_ipmi(host, arguments=ipmi_pxe_persistent, semaphore=new_semaphore) await ipmi_reset(host, new_semaphore) except Exception as ex: logger.debug(ex) logger.error( f"There was something wrong setting PXE flag or resetting IPMI on {host}." ) schedule = Schedule.current_schedule(cloud=_new_cloud_obj, host=_host_obj).first() if schedule: schedule.update(build_start=build_start, build_end=datetime.now()) schedule.save() logger.debug("Updating host: %s") _host_obj.update(cloud=_new_cloud_obj, build=False, last_build=datetime.now(), validated=False) return True
def main(): loop = asyncio.get_event_loop() foreman_admin = Foreman( conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"], loop=loop, ) ignore = ["cloud01"] foreman_rbac_exclude = conf.get("foreman_rbac_exclude") if foreman_rbac_exclude: ignore.extend(foreman_rbac_exclude.split("|")) clouds = Cloud.objects() for cloud in clouds: infra_pass = f"{conf['infra_location']}@{cloud.ticket}" loop.run_until_complete( foreman_admin.update_user_password(cloud.name, infra_pass) ) foreman_cloud_user = Foreman( conf["foreman_api_url"], cloud.name, infra_pass, loop=loop, ) if cloud.name not in ignore: logger.info(f"Processing {cloud.name}") cloud_hosts = loop.run_until_complete(foreman_cloud_user.get_all_hosts()) user_id = loop.run_until_complete(foreman_admin.get_user_id(cloud.name)) admin_id = loop.run_until_complete( foreman_admin.get_user_id(conf["foreman_username"]) ) current_schedule = Schedule.current_schedule(cloud=cloud) if current_schedule: logger.info(f" Current Host Permissions:") for host, properties in cloud_hosts.items(): logger.info(f" {host}") match = [ schedule.host.name for schedule in current_schedule if schedule.host.name == host ] if not match: _host_id = loop.run_until_complete( foreman_admin.get_host_id(host) ) loop.run_until_complete( foreman_admin.put_element( "hosts", _host_id, "owner_id", admin_id ) ) logger.info(f"* Removed permission {host}") for schedule in current_schedule: match = [ host for host, _ in cloud_hosts.items() if host == schedule.host.name ] if not match: # want to run these separately to avoid ServerDisconnect _host_id = loop.run_until_complete( foreman_admin.get_host_id(schedule.host.name) ) loop.run_until_complete( foreman_admin.put_element( "hosts", _host_id, "owner_id", user_id ) ) logger.info(f"* Added permission {schedule.host.name}") else: if cloud_hosts: logger.info(" No active schedule, removing pre-existing roles.") for host, properties in cloud_hosts.items(): _host_id = loop.run_until_complete( foreman_admin.get_host_id(host) ) loop.run_until_complete( foreman_admin.put_element( "hosts", _host_id, "owner_id", admin_id ) ) logger.info(f"* Removed permission {host}") else: logger.info(" No active schedule nor roles assigned.")
def POST(self, **data): # make sure post data passed in is ready to pass to mongo engine result, data = model.Schedule.prep_data(data) _start = None _end = None if "start" in data: _start = datetime.datetime.strptime(data["start"], "%Y-%m-%d %H:%M") if "end" in data: _end = datetime.datetime.strptime(data["end"], "%Y-%m-%d %H:%M") # check for broken hosts from foreman # to enable checking foreman host health before scheduling # set foreman_check_host_health: true in conf/quads.yml if conf["foreman_check_host_health"]: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) foreman = Foreman( conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"], loop=loop, ) broken_hosts = loop.run_until_complete(foreman.get_broken_hosts()) if broken_hosts.get(data["host"], False): result.append("Host %s is in broken state" % data["host"]) # Check if there were data validation errors if result: result = ["Data validation failed: %s" % ", ".join(result)] cherrypy.response.status = "400 Bad Request" return json.dumps({"result": result}) cloud_obj = None if "cloud" in data: cloud_obj = model.Cloud.objects(name=data["cloud"]).first() if not cloud_obj: result.append("Provided cloud does not exist") cherrypy.response.status = "400 Bad Request" return json.dumps({"result": result}) _host = data["host"] _host_obj = model.Host.objects(name=_host).first() if "index" in data: data["host"] = _host_obj schedule = self.model.objects( index=data["index"], host=data["host"] ).first() if schedule: if not _start: _start = schedule.start if not _end: _end = schedule.end if not cloud_obj: cloud_obj = schedule.cloud if model.Schedule.is_host_available( host=_host, start=_start, end=_end, exclude=schedule.index ): data["cloud"] = cloud_obj notification_obj = model.Notification.objects( cloud=cloud_obj, ticket=cloud_obj.ticket ).first() if notification_obj: notification_obj.update( one_day=False, three_days=False, five_days=False, seven_days=False, ) schedule.update(**data) result.append("Updated %s %s" % (self.name, schedule.index)) else: result.append("Host is not available during that time frame") else: try: if model.Schedule.is_host_available(host=_host, start=_start, end=_end): if self.model.current_schedule(cloud=cloud_obj) and cloud_obj.validated: if not cloud_obj.wipe: _host_obj.update(validated=True) notification_obj = model.Notification.objects( cloud=cloud_obj, ticket=cloud_obj.ticket ).first() if notification_obj: notification_obj.update(success=False) schedule = model.Schedule() data["cloud"] = cloud_obj schedule.insert_schedule(**data) cherrypy.response.status = "201 Resource Created" result.append( "Added schedule for %s on %s" % (data["host"], cloud_obj.name) ) else: result.append("Host is not available during that time frame") except Exception as e: # TODO: make sure when this is thrown the output # points back to here and gives the end user # enough information to fix the issue cherrypy.response.status = "500 Internal Server Error" result.append("Error: %s" % e) return json.dumps({"result": result})
async def post_system_test(self): password = f"{Config['infra_location']}@{self.cloud.ticket}" foreman = Foreman( Config["foreman_api_url"], self.cloud.name, password, loop=self.loop, ) valid_creds = await foreman.verify_credentials() if not valid_creds: logger.error("Unable to query Foreman for cloud: %s" % self.cloud.name) logger.error("Verify Foreman password is correct: %s" % password) self.report = ( self.report + "Unable to query Foreman for cloud: %s\n" % self.cloud.name ) self.report = ( self.report + "Verify Foreman password is correct: %s\n" % password ) return False build_hosts = await foreman.get_build_hosts() pending = [] schedules = Schedule.current_schedule(cloud=self.cloud) if schedules: for schedule in schedules: if schedule.host and schedule.host.name in build_hosts: pending.append(schedule.host.name) if pending: logger.info( "The following hosts are marked for build and will now be rebooted:" ) self.report = ( self.report + "The following hosts are marked for build:\n" ) for host in pending: logger.info(host) try: nc = Netcat(host) healthy = await nc.health_check() except OSError: healthy = False if not healthy: logger.warning( "Host %s didn't pass the health check. " "Potential provisioning in process. SKIPPING." % host ) continue badfish = None try: badfish = await badfish_factory( "mgmt-" + host, str(Config["ipmi_username"]), str(Config["ipmi_password"]), ) if is_supported(host): await badfish.boot_to_type( "foreman", os.path.join( os.path.dirname(__file__), "../../conf/idrac_interfaces.yml", ), ) else: await badfish.set_next_boot_pxe() await badfish.reboot_server() except BadfishException as ಥ﹏ಥ: logger.debug(ಥ﹏ಥ) if badfish: logger.warning( f"There was something wrong trying to boot from Foreman interface for: {host}" ) await badfish.reboot_server() else: logger.error( f"Could not initiate Badfish instance for: {host}" ) self.report = self.report + "%s\n" % host return False failed = False for host in self.hosts: try: badfish = await badfish_factory( "mgmt-" + host.name, str(Config["ipmi_cloud_username"]), password, ) await badfish.validate_credentials() except BadfishException: logger.info(f"Could not verify badfish credentials for: {host.name}") failed = True return not failed
def POST(self, **data): # make sure post data passed in is ready to pass to mongo engine result, data = model.Schedule.prep_data(data) _start = None _end = None if "start" in data: _start = datetime.datetime.strptime(data["start"], '%Y-%m-%d %H:%M') if "end" in data: _end = datetime.datetime.strptime(data["end"], '%Y-%m-%d %H:%M') # check for broken hosts from foreman # to enable checking foreman host health before scheduling # set foreman_check_host_health: true in conf/quads.yml if conf["foreman_check_host_health"]: foreman = Foreman(conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"]) broken_hosts = foreman.get_broken_hosts() if broken_hosts.get(data['host'], False): result.append("Host %s is in broken state" % data['host']) # Check if there were data validation errors if result: result = ['Data validation failed: %s' % ', '.join(result)] cherrypy.response.status = "400 Bad Request" return json.dumps({'result': result}) cloud_obj = None if "cloud" in data: cloud_obj = model.Cloud.objects(name=data["cloud"]).first() if not cloud_obj: result.append("Provided cloud does not exist") cherrypy.response.status = "400 Bad Request" return json.dumps({'result': result}) _host = data["host"] _host_obj = model.Host.objects(name=_host).first() if "index" in data: data["host"] = _host_obj schedule = self.model.objects(index=data["index"], host=data["host"]).first() if schedule: if not _start: _start = schedule.start if not _end: _end = schedule.end if not cloud_obj: cloud_obj = schedule.cloud if model.Schedule.is_host_available(host=_host, start=_start, end=_end, exclude=schedule.index): data["cloud"] = cloud_obj schedule.update(**data) result.append('Updated %s %s' % (self.name, schedule.index)) else: result.append( "Host is not available during that time frame") else: try: schedule = model.Schedule() if model.Schedule.is_host_available(host=_host, start=_start, end=_end): data["cloud"] = cloud_obj schedule.insert_schedule(**data) cherrypy.response.status = "201 Resource Created" result.append('Added schedule for %s on %s' % (data["host"], cloud_obj.name)) else: result.append( "Host is not available during that time frame") except Exception as e: # TODO: make sure when this is thrown the output # points back to here and gives the end user # enough information to fix the issue cherrypy.response.status = "500 Internal Server Error" result.append('Error: %s' % e) return json.dumps({'result': result})
badfish.change_boot(host_type, interfaces, True) except SystemExit as ex: logger.debug(ex) logger.error("Something went wrong.") return _host.update(build=False, realeased=True, last_build=datetime.now()) return else: _host.update(build=False, realeased=True, last_build=datetime.now()) return if __name__ == "__main__": foreman = Foreman( conf["foreman_api_url"], conf["foreman_username"], conf["foreman_password"] ) # get all overcloud hosts overclouds = foreman.get_parametrized("params.%s" % conf["foreman_director_parameter"], "true") logger.debug("Overclouds: %s" % overclouds) for host in overclouds: _quads_host = Host.objects(name=host).first() if _quads_host and not _quads_host.nullos: build_state = foreman.get_host_build_status(host) if build_state: _quads_host.update(nullos=True, build=True) # get all undercloud hosts underclouds = foreman.get_parametrized("params.%s" % conf["foreman_director_parameter"], "false")