def _config_required(config_path: Path) -> dict: try: with config_path.open("r") as cf: config = yaml.safe_load(cf) log.debug("Unvalidated data from file '{f}': {c}", f=str(config_path), c=config) except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error: raise ConfigError(str(yaml_error)) return config
async def check_redis_instance(): """Ensure Redis is running before starting server. Returns: {bool} -- True if Redis is running. """ await check_redis(db=params.cache.database, config=REDIS_CONFIG) log.debug( f"Redis is running at: {REDIS_CONFIG['host']}:{REDIS_CONFIG['port']}") return True
async def send(self, query): """Send an incoming webhook to http endpoint.""" payload = Webhook(**query) log.debug("Sending query data to {}:\n{}", self.config.host.host, payload) return await self._apost( endpoint=self.config.host.path, headers=self.config.headers, params=self.config.params, data=payload.export_dict(), )
def serialize(self): """Convert the Arista-formatted fields to standard parsed data model.""" routes = [] count = 0 for prefix, entries in self.bgp_route_entries.items(): count += entries.total_paths for route in entries.bgp_route_paths: as_path = self._get_as_path(route.as_path_entry.as_path) rpki_state = RPKI_STATE_MAP.get( route.route_type.origin_validity, 3) # BGP AS Path and BGP Community queries do not include the routeDetail # block. Therefore, we must verify it exists before including its data. communities = [] if route.route_detail is not None: communities = route.route_detail.community_list # iBGP paths contain an empty AS_PATH array. If the AS_PATH is empty, we # set the source_as to the router's local-as. source_as = self.asn if len(as_path) != 0: source_as = as_path[0] routes.append({ "prefix": prefix, "active": route.route_type.active, "age": self._get_route_age(route.timestamp), "weight": route.weight, "med": route.med, "local_preference": route.local_preference, "as_path": as_path, "communities": communities, "next_hop": route.next_hop, "source_as": source_as, "source_rid": route.peer_entry.peer_router_id, "peer_rid": route.peer_entry.peer_router_id, "rpki_state": rpki_state, }) serialized = ParsedRoutes( vrf=self.vrf, count=count, routes=routes, winning_weight=WINNING_WEIGHT, ) log.debug("Serialized Arista response: {}", serialized) return serialized
def generate_opengraph( image_path: Path, max_width: int, max_height: int, target_path: Path, background_color: str, ): """Generate an OpenGraph compliant image.""" from PIL import Image def center_point(background: Image, foreground: Image): """Generate a tuple of center points for PIL.""" bg_x, bg_y = background.size[0:2] fg_x, fg_y = foreground.size[0:2] x1 = math.floor((bg_x / 2) - (fg_x / 2)) y1 = math.floor((bg_y / 2) - (fg_y / 2)) x2 = math.floor((bg_x / 2) + (fg_x / 2)) y2 = math.floor((bg_y / 2) + (fg_y / 2)) return (x1, y1, x2, y2) # Convert image to JPEG format with static name "opengraph.jpg" dst_path = target_path / "opengraph.jpg" # Copy the original image to the target path copied = shutil.copy2(image_path, target_path) log.debug("Copied {} to {}", str(image_path), str(target_path)) with Image.open(copied) as src: # Only resize the image if it needs to be resized if src.size[0] != max_width or src.size[1] != max_height: # Resize image while maintaining aspect ratio log.debug("Opengraph image is not 1200x630, resizing...") src.thumbnail((max_width, max_height)) # Only impose a background image if the original image has # alpha/transparency channels if src.mode in ("RGBA", "LA"): log.debug("Opengraph image has transparency, converting...") background = Image.new("RGB", (max_width, max_height), background_color) background.paste(src, box=center_point(background, src)) dst = background else: dst = src # Save new image to derived target path dst.save(dst_path) # Delete the copied image Path(copied).unlink() if not dst_path.exists(): raise RuntimeError( f"Unable to save resized image to {str(dst_path)}") log.debug("Opengraph image ready at {}", str(dst_path)) return True
async def parsed_response( # noqa: C901 ("too complex") self, output: Iterable) -> str: """Send output through common parsers.""" log.debug("Pre-parsed responses:\n{}", output) parsed = () response = None structured_nos = structured_parsers.keys() structured_query_types = structured_parsers.get(self.device.nos, {}).keys() scrape_nos = scrape_parsers.keys() scrape_query_types = scrape_parsers.get(self.device.nos, {}).keys() if not self.device.structured_output: _parsed = () for func in parsers: for response in output: _output = func(commands=self.query, output=response) _parsed += (_output, ) if self.device.nos in scrape_nos and self.query_type in scrape_query_types: func = scrape_parsers[self.device.nos][self.query_type] for response in _parsed: _output = func(response) parsed += (_output, ) else: parsed += _parsed response = "\n\n".join(parsed) elif (self.device.structured_output and self.device.nos in structured_nos and self.query_type not in structured_query_types): for func in parsers: for response in output: _output = func(commands=self.query, output=response) parsed += (_output, ) response = "\n\n".join(parsed) elif (self.device.structured_output and self.device.nos in structured_nos and self.query_type in structured_query_types): func = structured_parsers[self.device.nos][self.query_type] response = func(output) if response is None: response = "\n\n".join(output) log.debug("Post-parsed responses:\n{}", response) return response
def msteams(self): """Format the webhook data as a Microsoft Teams card.""" def code(value): """Wrap argument in backticks for markdown inline code formatting.""" return f"`{str(value)}`" header_data = [ {"name": k, "value": code(v)} for k, v in self.headers.dict(by_alias=True).items() ] time_fmt = self.timestamp.strftime("%Y %m %d %H:%M:%S") payload = { "@type": "MessageCard", "@context": "http://schema.org/extensions", "themeColor": "118ab2", "summary": _WEBHOOK_TITLE, "sections": [ { "activityTitle": _WEBHOOK_TITLE, "activitySubtitle": f"{time_fmt} UTC", "activityImage": _ICON_URL, "facts": [ {"name": "Query Location", "value": self.query_location}, {"name": "Query Target", "value": code(self.query_target)}, {"name": "Query Type", "value": self.query_type}, {"name": "Query VRF", "value": self.query_vrf}, ], }, {"markdown": True, "text": "**Source Information**"}, {"markdown": True, "text": "---"}, { "markdown": True, "facts": [ {"name": "IP", "value": code(self.source)}, {"name": "Prefix", "value": code(self.network.prefix)}, {"name": "ASN", "value": code(self.network.asn)}, {"name": "Country", "value": self.network.country}, {"name": "Organization", "value": self.network.org}, ], }, {"markdown": True, "text": "**Request Headers**"}, {"markdown": True, "text": "---"}, {"markdown": True, "facts": header_data}, ], } log.debug("Created MS Teams webhook: {}", str(payload)) return payload
def _config_optional(config_path: Path) -> dict: if config_path is None: config = {} else: try: with config_path.open("r") as cf: config = yaml.safe_load(cf) or {} log.debug( "Unvalidated data from file '{f}': {c}", f=str(config_path), c=config, ) except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error: raise ConfigError(error_msg=str(yaml_error)) return config
def parse_juniper(output: Sequence) -> Dict: # noqa: C901 """Parse a Juniper BGP XML response.""" data = {} for i, response in enumerate(output): cleaned = clean_xml_output(response) try: parsed = xmltodict.parse( cleaned, force_list=("rt", "rt-entry", "community") ) log.debug("Initially Parsed Response: \n{}", parsed) if "rpc-reply" in parsed.keys(): parsed_base = parsed["rpc-reply"]["route-information"] elif "route-information" in parsed.keys(): parsed_base = parsed["route-information"] if "route-table" not in parsed_base: return data if "rt" not in parsed_base["route-table"]: return data parsed = parsed_base["route-table"] validated = JuniperRoute(**parsed) serialized = validated.serialize().export_dict() if i == 0: data.update(serialized) else: data["routes"].extend(serialized["routes"]) except xmltodict.expat.ExpatError as err: log.critical(str(err)) raise ParsingError("Error parsing response data") from err except KeyError as err: log.critical("{} was not found in the response", str(err)) raise ParsingError("Error parsing response data") except ValidationError as err: log.critical(str(err)) raise ParsingError(err.errors()) return data
async def _check_redis(): """Ensure Redis is running before starting server. Raises: HyperglassError: Raised if Redis is not running. Returns: {bool} -- True if Redis is running. """ try: await check_redis(db=params.cache.database, config=REDIS_CONFIG) except RuntimeError as e: raise HyperglassError(str(e), level="danger") from None log.debug(f"Redis is running at: {REDIS_CONFIG['host']}:{REDIS_CONFIG['port']}") return True
async def read_package_json() -> Dict: """Import package.json as a python dict.""" package_json_file = Path(__file__).parent.parent / "ui" / "package.json" try: with package_json_file.open("r") as file: package_json = json.load(file) except Exception as e: raise RuntimeError(f"Error reading package.json: {str(e)}") log.debug("package.json:\n{p}", p=package_json) return package_json
def queries(self): """Return queries for each enabled AFI. Returns: {list} -- List of queries to run """ query = [] for afi in self.afis: if self.transport == "rest": query.append(self.json(afi=afi)) else: query.append(self.scrape(afi=afi)) log.debug(f"Constructed query: {query}") return query
def json(self, afi): """Return JSON version of validated query for REST devices. Arguments: afi {object} -- AFI object Returns: {str} -- JSON query string """ log.debug("Building JSON query for {q}", q=repr(self.query_data)) return _json.dumps({ "query_type": self.query_data.query_type, "vrf": self.query_data.query_vrf.name, "afi": afi.protocol, "source": str(afi.source_address), "target": str(self.target), })
def _member_of(target, network): """Check if IP address belongs to network. Arguments: target {object} -- Target IPv4/IPv6 address network {object} -- ACL network Returns: {bool} -- True if target is a member of network, False if not """ log.debug(f"Checking membership of {target} for {network}") membership = False if (network.network_address <= target.network_address and network.broadcast_address >= target.broadcast_address # NOQA: W503 ): log.debug(f"{target} is a member of {network}") membership = True return membership
def resolve_hostname(hostname: str) -> Generator: """Resolve a hostname via DNS/hostfile.""" from socket import getaddrinfo, gaierror log.debug("Ensuring '{}' is resolvable...", hostname) ip4 = None ip6 = None try: res = getaddrinfo(hostname, None) for sock in res: if sock[0].value == 2 and ip4 is None: ip4 = ip_address(sock[4][0]) elif sock[0].value == 30 and ip6 is None: ip6 = ip_address(sock[4][0]) except (gaierror, ValueError, IndexError): pass yield ip4 yield ip6
def network_info_sync(*targets: str) -> Dict[str, Dict[str, str]]: """Get ASN, Containing Prefix, and other info about an internet resource.""" targets = [str(t) for t in targets] cache = SyncCache(db=params.cache.database, **REDIS_CONFIG) # Set default data structure. data = {t: {k: "" for k in DEFAULT_KEYS} for t in targets} # Get all cached bgp.tools data. cached = cache.get_dict(CACHE_KEY) # Try to use cached data for each of the items in the list of # resources. for t in targets: if t in cached: # Reassign the cached network info to the matching resource. data[t] = cached[t] log.debug("Using cached network info for {}", t) # Remove cached items from the resource list so they're not queried. targets = [t for t in targets if t not in cached] try: if targets: whoisdata = run_whois_sync(targets) if whoisdata: # If the response is not empty, parse it. data.update(parse_whois(whoisdata, targets)) # Cache the response for t in targets: cache.set_dict(CACHE_KEY, t, data[t]) log.debug("Cached network info for {}", t) except Exception as err: log.error(str(err)) return data
async def response(self): """Initiate query validation and execution.""" device = getattr(devices, self.query_location) log.debug(f"Received query for {self.query_data}") log.debug(f"Matched device config: {device}") supported, transport = validate_nos(device.nos) connect = None output = params.messages.general connect = Connect(device, self.query_data, transport) if supported and transport == "rest": output = await connect.rest() elif supported and transport == "scrape": if device.proxy: output = await connect.scrape_proxied() else: output = await connect.scrape_direct() else: raise ConfigError('"{nos}" is not supported.', nos=device.nos) if output == "" or output == "\n": raise ResponseEmpty(params.messages.no_output, device_name=device.display_name) log.debug( f"Output for query: {self.query_data.json()}:\n{repr(output)}") return output
async def network_info(self, resource, serialize=False): """Get network info via RIPE's Network Info API endpoint. See: https://stat.ripe.net/docs/data_api#network-info """ try: valid_ip = ip_address(resource) if not valid_ip.is_global: log.debug("IP {ip} is not a global address", ip=str(valid_ip)) return {"prefix": None, "asn": None} except ValueError: log.debug("'{resource}' is not a valid IP address", resource=resource) return {"prefix": None, "asn": None} raw = await self._aget(endpoint="network-info", params={"resource": valid_ip}) data = { "asns": raw["data"]["asns"], "prefix": ip_network(raw["data"]["prefix"]), } if serialize: data["prefix"] = str(data["prefix"]) data["asns"] = data["asns"][0] log.debug("Collected network info from RIPEState: {i}", i=str(data)) return data
def serialize(self): """Convert the Juniper-specific fields to standard parsed data model.""" vrf_parts = self.table_name.split(".") if len(vrf_parts) == 2: vrf = "default" else: vrf = vrf_parts[0] routes = [] count = 0 for table in self.rt: count += table.rt_entry_count prefix = "/".join( str(i) for i in (table.rt_destination, table.rt_prefix_length)) for route in table.rt_entry: routes.append({ "prefix": prefix, "active": route.active_tag, "age": route.age, "weight": route.preference, "med": route.metric, "local_preference": route.local_preference, "as_path": route.as_path, "communities": route.communities, "next_hop": route.next_hop, "source_as": route.source_as, "source_rid": route.source_rid, "peer_rid": route.peer_rid, "rpki_state": route.validation_state, }) serialized = ParsedRoutes( vrf=vrf, count=count, routes=routes, winning_weight="low", ) log.debug("Serialized Juniper response: {}", serialized) return serialized
def resolve_hostname(hostname: str) -> Generator: """Resolve a hostname via DNS/hostfile.""" # Standard Library from socket import gaierror, getaddrinfo log.debug("Ensuring '{}' is resolvable...", hostname) ip4 = None ip6 = None try: res = getaddrinfo(hostname, None) for sock in res: if sock[0].value == 2 and ip4 is None: ip4 = ip_address(sock[4][0]) elif sock[0].value in (10, 30) and ip6 is None: ip6 = ip_address(sock[4][0]) except (gaierror, ValueError, IndexError) as err: log.debug(str(err)) pass yield ip4 yield ip6
async def import_certificate(encoded_request: EncodedRequest): """Import a certificate from hyperglass-agent.""" # Try to match the requested device name with configured devices log.debug("Attempting certificate import for device '{}'", devices[encoded_request.device]) try: matched_device = devices[encoded_request.device] except AttributeError: raise HTTPException( detail=f"Device {str(encoded_request.device)} not found", status_code=404) try: # Decode JSON Web Token decoded_request = await jwt_decode( payload=encoded_request.encoded, secret=matched_device.credential.password.get_secret_value(), ) except HyperglassError as decode_error: raise HTTPException(detail=str(decode_error), status_code=400) try: # Write certificate to file import_public_key( app_path=APP_PATH, device_name=matched_device._id, keystring=decoded_request, ) except RuntimeError as err: raise HyperglassError(str(err), level="danger") log.info("Added public key for {}", encoded_request.device) return { "output": f"Added public key for {encoded_request.device}", "level": "success", "keywords": [encoded_request.device], }
def run_whois_sync(resource: str): """Open raw socket to bgp.tools and execute query.""" # Open the socket to bgp.tools log.debug("Opening connection to bgp.tools") sock = socket.socket() sock.connect(("bgp.tools", 43)) sock.send(f"{resource}\n".encode()) # Read the response response = b"" while True: data = sock.recv(128) if data: response += data else: log.debug("Closing connection to bgp.tools") sock.shutdown(1) sock.close() break return response.decode()
def parse_whois(output: str): """Parse raw whois output from bgp.tools. Sample output: AS | IP | BGP Prefix | CC | Registry | Allocated | AS Name # noqa: E501 13335 | 1.1.1.1 | 1.1.1.0/24 | US | ARIN | 2010-07-14 | Cloudflare, Inc. """ # Each new line is a row. rawlines = output.split("\n") lines = () for rawline in rawlines: # Split each row into fields, separated by a pipe. line = () rawfields = rawline.split("|") for rawfield in rawfields: # Remove newline and leading/trailing whitespaces. field = re.sub(r"(\n|\r)", "", rawfield).strip(" ") line += (field, ) lines += (line, ) headers = lines[0] row = lines[1] data = {} for i, header in enumerate(headers): # Try to replace bgp.tools key names with easier to parse key names key = REPLACE_KEYS.get(header, header) data.update({key: row[i]}) log.debug("Parsed bgp.tools data: {}", data) return data
def parse_whois(output: str, targets: List[str]) -> Dict[str, str]: """Parse raw whois output from bgp.tools. Sample output: AS | IP | BGP Prefix | CC | Registry | Allocated | AS Name 13335 | 1.1.1.1 | 1.1.1.0/24 | US | ARIN | 2010-07-14 | Cloudflare, Inc. """ def lines(raw): """Generate clean string values for each column.""" for r in (r for r in raw.split("\n") if r): fields = (re.sub(r"(\n|\r)", "", field).strip(" ") for field in r.split("|")) yield fields data = {} for line in lines(output): # Unpack each line's parsed values. asn, ip, prefix, country, rir, allocated, org = line # Match the line to the item in the list of resources to query. if ip in targets: i = targets.index(ip) data[targets[i]] = { "asn": asn, "ip": ip, "prefix": prefix, "country": country, "rir": rir, "allocated": allocated, "org": org, } log.debug("Parsed bgp.tools data: {}", data) return data
def rpki_state(prefix, asn): """Get RPKI state and map to expected integer.""" log.debug("Validating RPKI State for {p} via AS{a}", p=prefix, a=asn) state = 3 ro = f"{prefix}@{asn}" cached = cache.get_dict(CACHE_KEY, ro) if cached is not None: state = cached else: ql = 'query GetValidation {{ validation(prefix: "{}", asn: {}) {{ state }} }}' query = ql.format(prefix, asn) try: with BaseExternal( base_url="https://rpki.cloudflare.com") as client: response = client._post("/api/graphql", data={"query": query}) validation_state = (response.get("data", {}).get("validation", {}).get( "state", "DEFAULT")) state = RPKI_STATE_MAP[validation_state] cache.set_dict(CACHE_KEY, ro, state) except Exception as err: log.error(str(err)) state = 3 msg = "RPKI Validation State for {} via AS{} is {}".format( prefix, asn, RPKI_NAME_MAP[state]) if cached is not None: msg += " [CACHED]" log.debug(msg) return state
async def execute(query: Query) -> Union[str, Sequence[Dict]]: """Initiate query validation and execution.""" output = params.messages.general log.debug("Received query for {}", query.json()) log.debug("Matched device config: {}", query.device) mapped_driver = map_driver(query.device.driver) driver = mapped_driver(query.device, query) timeout_args = { "unformatted_msg": params.messages.connection_error, "device_name": query.device.name, "error": params.messages.request_timeout, } if query.device.proxy: timeout_args["proxy"] = query.device.proxy.name signal.signal(signal.SIGALRM, handle_timeout(**timeout_args)) signal.alarm(params.request_timeout - 1) if query.device.proxy: proxy = driver.setup_proxy() with proxy() as tunnel: response = await driver.collect( tunnel.local_bind_host, tunnel.local_bind_port ) else: response = await driver.collect() output = await driver.parsed_response(response) if isinstance(output, str): # If the output is a string (not structured) and is empty, # produce an error. if output == "" or output == "\n": raise ResponseEmpty( params.messages.no_output, device_name=query.device.name ) elif isinstance(output, Dict): # If the output an empty dict, responses have data, produce an # error. if not output: raise ResponseEmpty( params.messages.no_output, device_name=query.device.name ) log.debug("Output for query: {}:\n{}", query.json(), repr(output)) signal.alarm(0) return output
async def execute(query: Query) -> Union[str, Sequence[Dict]]: """Initiate query validation and execution.""" output = params.messages.general log.debug("Received query for {}", query.json()) log.debug("Matched device config: {}", query.device) supported, driver_name = validate_nos(query.device.nos) mapped_driver = DRIVER_MAP.get(driver_name, NetmikoConnection) driver = mapped_driver(query.device, query) timeout_args = { "unformatted_msg": params.messages.connection_error, "device_name": query.device.name, "error": params.messages.request_timeout, } if query.device.proxy: timeout_args["proxy"] = query.device.proxy.name signal.signal(signal.SIGALRM, handle_timeout(**timeout_args)) signal.alarm(params.request_timeout - 1) if query.device.proxy: proxy = driver.setup_proxy() with proxy() as tunnel: response = await driver.collect( tunnel.local_bind_host, tunnel.local_bind_port ) else: response = await driver.collect() output = await driver.parsed_response(response) if output == "" or output == "\n": raise ResponseEmpty(params.messages.no_output, device_name=query.device.name) log.debug("Output for query: {}:\n{}", query.json(), repr(output)) signal.alarm(0) return output
def parse_arista(output: Sequence[str]) -> Dict: # noqa: C901 """Parse a Arista BGP JSON response.""" data = {} for i, response in enumerate(output): try: data: Dict = json.loads(response) log.debug("Pre-parsed data: {}", data) vrf = list(data["vrfs"].keys())[0] routes = data["vrfs"][vrf] log.debug("Pre-validated data: {}", routes) validated = AristaRoute(**routes) serialized = validated.serialize().export_dict() if i == 0: data.update(serialized) else: data["routes"].extend(serialized["routes"]) except json.JSONDecodeError as err: log.critical("Error decoding JSON: {}", str(err)) raise ParsingError("Error parsing response data") except KeyError as err: log.critical("'{}' was not found in the response", str(err)) raise ParsingError("Error parsing response data") except IndexError as err: log.critical(str(err)) raise ParsingError("Error parsing response data") except ValidationError as err: log.critical(str(err)) raise ParsingError(err.errors()) log.debug("Serialzed: {}", data) return data
logger=log, log_directory=params.logging.directory, log_format=params.logging.format, log_max_size=params.logging.max_size, ) # Set up syslog logging if enabled. if params.logging.syslog is not None and params.logging.syslog.enable: enable_syslog_logging( logger=log, syslog_host=params.logging.syslog.host, syslog_port=params.logging.syslog.port, ) if params.logging.http is not None and params.logging.http.enable: log.debug("HTTP logging is enabled") # Perform post-config initialization string formatting or other # functions that require access to other config levels. E.g., # something in 'params.web.text' needs to be formatted with a value # from params. try: params.web.text.subtitle = params.web.text.subtitle.format(**params.dict( exclude={"web", "queries", "messages"})) # If keywords are unmodified (default), add the org name & # site_title. if _params.Params().site_keywords == params.site_keywords: params.site_keywords = sorted( {*params.site_keywords, params.org_name, params.site_title})
async def __aexit__(self, exc_type=None, exc_value=None, traceback=None): """Close connection on exit.""" log.debug("Closing session with {}", self.base_url) await self._asession.aclose() return True