def on_starting(server: Arbiter): """Gunicorn pre-start tasks.""" setup_lib_logging(params.debug) python_version = platform.python_version() required = ".".join((str(v) for v in MIN_PYTHON_VERSION)) log.info("Python {} detected ({} required)", python_version, required) async def runner(): # Standard Library from asyncio import gather await gather(build_ui(), cache_config()) check_redis_instance() aiorun(build_ui()) cache_config() log.success( "Started hyperglass {v} on http://{h}:{p} with {w} workers", v=__version__, h=format_listen_address(params.listen_address), p=str(params.listen_port), w=server.app.cfg.settings["workers"].value, )
async def build_ui(app_path): """Execute `next build` & `next export` from UI directory. Raises: RuntimeError: Raised if exit code is not 0. RuntimeError: Raised when any other error occurs. """ import os import asyncio try: timeout = os.environ["HYPERGLASS_UI_BUILD_TIMEOUT"] log.info("Found UI build timeout environment variable: {}", timeout) timeout = int(timeout) except KeyError: timeout = 90 ui_dir = Path(__file__).parent / "ui" build_dir = app_path / "static" / "ui" build_command = "node_modules/.bin/next build" export_command = "node_modules/.bin/next export -o {f}".format(f=build_dir) all_messages = [] for command in (build_command, export_command): try: proc = await asyncio.create_subprocess_shell( cmd=command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, cwd=ui_dir, ) stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) messages = stdout.decode("utf-8").strip() errors = stderr.decode("utf-8").strip() if proc.returncode != 0: raise RuntimeError( f"\nMessages:\n{messages}\nErrors:\n{errors}") await proc.wait() all_messages.append(messages) except asyncio.TimeoutError: raise RuntimeError( f"{timeout} second timeout exceeded while building UI") except Exception as e: raise RuntimeError(str(e)) return "\n".join(all_messages)
def get_ui_build_timeout() -> Optional[int]: """Read the UI build timeout from environment variables or set a default.""" timeout = None if "HYPERGLASS_UI_BUILD_TIMEOUT" in os.environ: timeout = int(os.environ["HYPERGLASS_UI_BUILD_TIMEOUT"]) log.info("Found UI build timeout environment variable: {}", timeout) elif "POETRY_HYPERGLASS_UI_BUILD_TIMEOUT" in os.environ: timeout = int(os.environ["POETRY_HYPERGLASS_UI_BUILD_TIMEOUT"]) log.info("Found UI build timeout environment variable: {}", timeout) return timeout
def run(): """Run tests.""" samples = () if len(sys.argv) == 2: samples += (sys.argv[1], ) else: for sample_file in SAMPLE_FILES: with sample_file.open("r") as file: samples += (file.read(), ) for sample in samples: parsed = parse_juniper([sample]) log.info(json.dumps(parsed, indent=2)) sys.exit(0)
def __init__( self, message: str = "", level: str = "warning", keywords: Optional[List[str]] = None, ) -> None: """Initialize the hyperglass base exception class.""" self._message = message self._level = level self._keywords = keywords or [] if self._level == "warning": log.error(repr(self)) elif self._level == "danger": log.critical(repr(self)) else: log.info(repr(self))
def __init__(self, message="", level="warning", keywords=None): """Initialize the hyperglass base exception class. Keyword Arguments: message {str} -- Error message (default: {""}) level {str} -- Error severity (default: {"warning"}) keywords {list} -- 'Important' keywords (default: {None}) """ self._message = message self._level = level self._keywords = keywords or [] if self._level == "warning": log.error(repr(self)) elif self._level == "danger": log.critical(repr(self)) else: log.info(repr(self))
def serialize(self): """Convert the Juniper-specific fields to standard parsed data model.""" vrf_parts = self.table_name.split(".") if len(vrf_parts) == 2: vrf = "default" else: vrf = vrf_parts[0] routes = [] count = 0 for table in self.rt: count += table.rt_entry_count prefix = "/".join( str(i) for i in (table.rt_destination, table.rt_prefix_length) ) for route in table.rt_entry: routes.append( { "prefix": prefix, "active": route.active_tag, "age": route.age, "weight": route.preference, "med": route.metric, "local_preference": route.local_preference, "as_path": route.as_path, "communities": route.communities, "next_hop": route.next_hop, "source_as": route.source_as, "source_rid": route.source_rid, "peer_rid": route.peer_rid, "rpki_state": route.validation_state, } ) serialized = ParsedRoutes( vrf=vrf, count=count, routes=routes, winning_weight="low", ) log.info("Serialized Juniper response: {}", serialized) return serialized
def __init__( self, message: str = "", level: str = "warning", keywords: Optional[List[str]] = None, ) -> None: """Initialize the hyperglass base exception class.""" self._message = message self._level = level self._keywords = keywords or [] if self._level == "warning": log.error(repr(self)) elif self._level == "danger": log.critical(repr(self)) else: log.info(repr(self)) if all(sys.exc_info()): # Rich will raise a ValueError if print_exception() is used # outside of a try/except block. Only use Rich for traceback # printing if the exception is caught. console.print_exception(extra_lines=6)
async def import_certificate(encoded_request: EncodedRequest): """Import a certificate from hyperglass-agent.""" # Try to match the requested device name with configured devices log.debug("Attempting certificate import for device '{}'", devices[encoded_request.device]) try: matched_device = devices[encoded_request.device] except AttributeError: raise HTTPException( detail=f"Device {str(encoded_request.device)} not found", status_code=404) try: # Decode JSON Web Token decoded_request = await jwt_decode( payload=encoded_request.encoded, secret=matched_device.credential.password.get_secret_value(), ) except HyperglassError as decode_error: raise HTTPException(detail=str(decode_error), status_code=400) try: # Write certificate to file import_public_key( app_path=APP_PATH, device_name=matched_device._id, keystring=decoded_request, ) except RuntimeError as err: raise HyperglassError(str(err), level="danger") log.info("Added public key for {}", encoded_request.device) return { "output": f"Added public key for {encoded_request.device}", "level": "success", "keywords": [encoded_request.device], }
def serialize(self): """Convert the FRR-specific fields to standard parsed data model.""" # TODO: somehow, get the actual VRF vrf = "default" routes = [] for route in self.paths: now = datetime.utcnow().timestamp() then = datetime.utcfromtimestamp(route.last_update).timestamp() age = int(now - then) routes.append({ "prefix": self.prefix, "active": route.bestpath, "age": age, "weight": route.weight, "med": route.med, "local_preference": route.localpref, "as_path": route.aspath, "communities": route.community, "next_hop": route.nexthops[0].ip, "source_as": route.aggregator_as, "source_rid": route.aggregator_id, "peer_rid": route.peer.peer_id, # TODO: somehow, get the actual RPKI state "rpki_state": 3, }) serialized = ParsedRoutes( vrf=vrf, count=len(routes), routes=routes, winning_weight="high", ) log.info("Serialized FRR response: {}", serialized) return serialized
DEFAULT_DETAILS, SUPPORTED_QUERY_TYPES, PARSED_RESPONSE_FIELDS, SUPPORTED_STRUCTURED_OUTPUT, __version__, ) from hyperglass.exceptions import ConfigError, ConfigInvalid, ConfigMissing from hyperglass.configuration.models import params as _params from hyperglass.configuration.models import routers as _routers from hyperglass.configuration.models import commands as _commands from hyperglass.configuration.markdown import get_markdown set_app_path(required=True) CONFIG_PATH = Path(os.environ["hyperglass_directory"]) log.info("Configuration directory: {d}", d=str(CONFIG_PATH)) # Project Directories WORKING_DIR = Path(__file__).resolve().parent CONFIG_FILES = ( ("hyperglass.yaml", False), ("devices.yaml", True), ("commands.yaml", False), ) def _check_config_files(directory): """Verify config files exist and are readable. Arguments: directory {Path} -- Config directory Path object
async def build_frontend( # noqa: C901 dev_mode: bool, dev_url: str, prod_url: str, params: dict, app_path: Path, force: bool = False, ): """Perform full frontend UI build process. Securely creates temporary file, writes frontend configuration parameters to file as JSON. Then writes the name of the temporary file to /tmp/hyperglass.env.json as {"configFile": <file_name> }. Webpack reads /tmp/hyperglass.env.json, loads the temporary file, and sets its contents to Node environment variables during the build process. After the build is successful, the temporary file is automatically closed during garbage collection. Arguments: dev_mode {bool} -- Development Mode dev_url {str} -- Development Mode URL prod_url {str} -- Production Mode URL params {dict} -- Frontend Config paramters Raises: RuntimeError: Raised if errors occur during build process. Returns: {bool} -- True if successful """ import hashlib import tempfile from aiofile import AIOFile import json from hyperglass.constants import __version__ env_file = Path("/tmp/hyperglass.env.json") # noqa: S108 package_json = await read_package_json() env_vars = { "_HYPERGLASS_CONFIG_": params, "_HYPERGLASS_VERSION_": __version__, "_HYPERGLASS_PACKAGE_JSON_": package_json, "_HYPERGLASS_APP_PATH_": str(app_path), } # Set NextJS production/development mode and base URL based on # developer_mode setting. if dev_mode: env_vars.update({ "NODE_ENV": "development", "_HYPERGLASS_URL_": dev_url }) else: env_vars.update({ "NODE_ENV": "production", "_HYPERGLASS_URL_": prod_url }) # Check if hyperglass/ui/node_modules has been initialized. If not, # initialize it. initialized = await check_node_modules() if initialized: log.debug("node_modules is already initialized") elif not initialized: log.debug( "node_modules has not been initialized. Starting initialization..." ) node_setup = await node_initial(dev_mode) if node_setup == "": log.debug("Re-initialized node_modules") try: env_json = json.dumps(env_vars, default=str) # Create SHA256 hash from all parameters passed to UI, use as # build identifier. build_id = hashlib.sha256(env_json.encode()).hexdigest() # Read hard-coded environment file from last build. If build ID # matches this build's ID, don't run a new build. if env_file.exists() and not force: async with AIOFile(env_file, "r") as ef: ef_json = await ef.read() ef_id = json.loads(ef_json).get("buildId", "empty") log.debug("Previous Build ID: {id}", id=ef_id) if ef_id == build_id: log.debug( "UI parameters unchanged since last build, skipping UI build..." ) return True # Create temporary file. json file extension is added for easy # webpack JSON parsing. temp_file = tempfile.NamedTemporaryFile(mode="w+", prefix="hyperglass_", suffix=".json", delete=not dev_mode) log.info("Starting UI build...") log.debug( f"Created temporary UI config file: '{temp_file.name}' for build {build_id}" ) async with AIOFile(temp_file.name, "w+") as temp: await temp.write(env_json) await temp.fsync() # Write "permanent" file (hard-coded named) for Node to read. async with AIOFile(env_file, "w+") as ef: await ef.write( json.dumps({ "configFile": temp_file.name, "buildId": build_id })) await ef.fsync() # While temporary file is still open, initiate UI build process. if not dev_mode or force: initialize_result = await node_initial(dev_mode) build_result = await build_ui(app_path=app_path) if initialize_result: log.debug(initialize_result) elif initialize_result == "": log.debug("Re-initialized node_modules") if build_result: log.success("Completed UI build") elif dev_mode and not force: log.debug( "Running in developer mode, did not build new UI files" ) await migrate_images(app_path, params) generate_opengraph( Path(params["web"]["opengraph"]["image"]), 1200, 630, app_path / "static" / "images", params["web"]["theme"]["colors"]["black"], ) except Exception as e: raise RuntimeError(str(e)) from None return True
async def query(query_data: Query, request: Request, background_tasks: BackgroundTasks): """Ingest request data pass it to the backend application to perform the query.""" timestamp = datetime.utcnow() background_tasks.add_task(send_webhook, query_data, request, timestamp) # Initialize cache cache = AsyncCache(db=params.cache.database, **REDIS_CONFIG) log.debug("Initialized cache {}", repr(cache)) # Use hashed query_data string as key for for k/v cache store so # each command output value is unique. cache_key = query_data.digest() # Define cache entry expiry time cache_timeout = params.cache.timeout log.debug("Cache Timeout: {}", cache_timeout) log.info("Starting query execution for query {}", query_data.summary) cache_response = await cache.get_dict(cache_key, "output") json_output = False if query_data.device.structured_output and query_data.query_type in ( "bgp_route", "bgp_community", "bgp_aspath", ): json_output = True cached = False runtime = 65535 if cache_response: log.debug("Query {} exists in cache", cache_key) # If a cached response exists, reset the expiration time. await cache.expire(cache_key, seconds=cache_timeout) cached = True runtime = 0 timestamp = await cache.get_dict(cache_key, "timestamp") elif not cache_response: log.debug("No existing cache entry for query {}", cache_key) log.debug("Created new cache key {} entry for query {}", cache_key, query_data.summary) timestamp = query_data.timestamp starttime = time.time() if params.fake_output: # Return fake, static data for development purposes, if enabled. cache_output = await fake_output(json_output) else: # Pass request to execution module cache_output = await execute(query_data) endtime = time.time() elapsedtime = round(endtime - starttime, 4) log.debug("Query {} took {} seconds to run.", cache_key, elapsedtime) if cache_output is None: raise HyperglassError(message=params.messages.general, alert="danger") # Create a cache entry if json_output: raw_output = json.dumps(cache_output) else: raw_output = str(cache_output) await cache.set_dict(cache_key, "output", raw_output) await cache.set_dict(cache_key, "timestamp", timestamp) await cache.expire(cache_key, seconds=cache_timeout) log.debug("Added cache entry for query: {}", cache_key) runtime = int(round(elapsedtime, 0)) # If it does, return the cached entry cache_response = await cache.get_dict(cache_key, "output") response_format = "text/plain" if json_output: response_format = "application/json" log.debug("Cache match for {}:\n{}", cache_key, cache_response) log.success("Completed query execution for query {}", query_data.summary) return { "output": cache_response, "id": cache_key, "cached": cached, "runtime": runtime, "timestamp": timestamp, "format": response_format, "random": query_data.random(), "level": "success", "keywords": [], }
"""Test Arista JSON Parsing.""" # Standard Library import sys import json from pathlib import Path # Project from hyperglass.log import log # Local from .arista import parse_arista SAMPLE_FILE = Path(__file__).parent.parent / "models" / "parsing" / "arista_route.json" if __name__ == "__main__": if len(sys.argv) == 2: sample = sys.argv[1] else: with SAMPLE_FILE.open("r") as file: sample = file.read() parsed = parse_arista([sample]) log.info(json.dumps(parsed, indent=2)) sys.exit(0)
async def query(query_data: Query, request: Request, background_tasks: BackgroundTasks): """Ingest request data pass it to the backend application to perform the query.""" timestamp = datetime.utcnow() background_tasks.add_task(send_webhook, query_data, request, timestamp) # Initialize cache cache = Cache(db=params.cache.database, **REDIS_CONFIG) log.debug("Initialized cache {}", repr(cache)) # Use hashed query_data string as key for for k/v cache store so # each command output value is unique. cache_key = query_data.digest() # Define cache entry expiry time cache_timeout = params.cache.timeout log.debug(f"Cache Timeout: {cache_timeout}") log.info(f"Starting query execution for query {query_data.summary}") cache_response = await cache.get_dict(cache_key, "output") cached = False if cache_response: log.debug("Query {q} exists in cache", q=cache_key) # If a cached response exists, reset the expiration time. await cache.expire(cache_key, seconds=cache_timeout) cached = True runtime = 0 timestamp = await cache.get_dict(cache_key, "timestamp") elif not cache_response: log.debug(f"No existing cache entry for query {cache_key}") log.debug( f"Created new cache key {cache_key} entry for query {query_data.summary}" ) timestamp = query_data.timestamp # Pass request to execution module starttime = time.time() cache_output = await Execute(query_data).response() endtime = time.time() elapsedtime = round(endtime - starttime, 4) log.debug(f"Query {cache_key} took {elapsedtime} seconds to run.") if cache_output is None: raise HyperglassError(message=params.messages.general, alert="danger") # Create a cache entry if query_data.device.structured_output: raw_output = json.dumps(cache_output) else: raw_output = str(cache_output) await cache.set_dict(cache_key, "output", raw_output) await cache.set_dict(cache_key, "timestamp", timestamp) await cache.expire(cache_key, seconds=cache_timeout) log.debug(f"Added cache entry for query: {cache_key}") runtime = int(round(elapsedtime, 0)) # If it does, return the cached entry cache_response = await cache.get_dict(cache_key, "output") if query_data.device.structured_output: response_format = "application/json" cache_response = json.loads(cache_response) else: response_format = "text/plain" log.debug(f"Cache match for {cache_key}:\n {cache_response}") log.success(f"Completed query execution for {query_data.summary}") return { "output": cache_response, "id": cache_key, "cached": cached, "runtime": runtime, "timestamp": timestamp, "format": response_format, "random": query_data.random(), "level": "success", "keywords": [], }